Compare commits

...

7 Commits

Author SHA1 Message Date
github-actions[bot] 7a2485cca2 Release 0.11.12 (#2050)
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>
2025-07-02 11:41:55 +07:00
Marcus Schiesser 1329186a23 docs: clarify how to run docs 2025-07-02 11:33:48 +07:00
dependabot[bot] 5d6e7384f5 chore(deps-dev): bump @modelcontextprotocol/server-filesystem from 2025.3.28 to 2025.7.1 (#2055) 2025-07-02 11:26:18 +07:00
allen f2dfd305fb implement bm25 retriever (#2045)
Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>
2025-07-02 11:22:47 +07:00
Huu Le 3cd8a573df feat: update interpreter to always upload all files in the configured directory (#2057) 2025-07-02 10:57:04 +07:00
Laurie Voss 09c6077f6e Import path for llamaparsereader (#2056) 2025-07-01 16:51:25 -07:00
Logan 14cc65b4e3 add google analytics (#2053)
Co-authored-by: Alex Yang <himself65@outlook.com>
2025-07-01 11:18:14 -07:00
144 changed files with 1925 additions and 667 deletions
-5
View File
@@ -1,5 +0,0 @@
---
"@llamaindex/core": patch
---
Add new memory API
-5
View File
@@ -1,5 +0,0 @@
---
"@llamaindex/core": patch
---
Fix: split sentences must not trim whitespaces
-6
View File
@@ -1,6 +0,0 @@
---
"llamaindex": patch
"@llamaindex/core": patch
---
Fix logging for fromPersistPath
+1 -1
View File
@@ -25,7 +25,7 @@ Make sure you have Node.js LTS (Long-term Support) installed. You can check your
```shell
node -v
# v20.x.x
# v22.x.x
```
### Use pnpm
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/doc
## 0.2.32
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- llamaindex@0.11.12
- @llamaindex/cloud@4.0.17
- @llamaindex/node-parser@2.0.13
- @llamaindex/openai@0.4.7
- @llamaindex/readers@3.1.12
- @llamaindex/workflow@1.1.13
## 0.2.31
### Patch Changes
+2
View File
@@ -3,6 +3,8 @@
This is a Next.js application generated with
[Create Fumadocs](https://github.com/fuma-nama/fumadocs).
> Note: Before running the development server, make sure to build the whole project first, see [CONTRIBUTING.md](../../CONTRIBUTING.md) for more details.
Run development server:
```bash
+3 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.2.31",
"version": "0.2.32",
"private": true,
"scripts": {
"postinstall": "fumadocs-mdx",
@@ -15,7 +15,6 @@
"dependencies": {
"@huggingface/transformers": "^3.5.0",
"@icons-pack/react-simple-icons": "^10.1.0",
"@llamaindex/workflow-docs": "0.1.1",
"@llamaindex/chat-ui-docs": "^0.0.5",
"@llamaindex/cloud": "workspace:*",
"@llamaindex/core": "workspace:*",
@@ -23,8 +22,10 @@
"@llamaindex/openai": "workspace:*",
"@llamaindex/readers": "workspace:*",
"@llamaindex/workflow": "workspace:*",
"@llamaindex/workflow-docs": "0.1.1",
"@mdx-js/mdx": "^3.1.0",
"@monaco-editor/react": "^4.7.0",
"@next/third-parties": "^15.3.4",
"@number-flow/react": "^0.3.4",
"@radix-ui/react-dialog": "^1.1.2",
"@radix-ui/react-icons": "^1.3.2",
+2
View File
@@ -1,5 +1,6 @@
import { AIProvider } from "@/actions";
import { TooltipProvider } from "@/components/ui/tooltip";
import { GoogleAnalytics } from "@next/third-parties/google";
import { RootProvider } from "fumadocs-ui/provider";
import { Inter } from "next/font/google";
import type { ReactNode } from "react";
@@ -39,6 +40,7 @@ export default function Layout({ children }: { children: ReactNode }) {
</AIProvider>
</TooltipProvider>
</body>
<GoogleAnalytics gaId="G-NB9B8LW9W5" />
</html>
);
}
@@ -53,7 +53,7 @@ console.log("Response:", result.data.result);
You can also put messages in Vercel format directly to the memory:
```ts twoslash
```ts
await memory.add({
id: "1",
createdAt: new Date(),
@@ -73,7 +73,7 @@ await memory.add({
If you call `get`, messages are usually retrieved in the LlamaIndexTS format (type `ChatMessage`). If you specify the `type` parameter using `get`, you can return the messages in different formats. E.g.: using `type: "vercel"`, you can return the messages in Vercel format:
```ts twoslash
```ts
const messages = await memory.get({ type: "vercel" });
console.log(messages);
```
@@ -95,7 +95,7 @@ For initialization, you call `createMemory` with the following options:
Example:
```ts twoslash
```ts
const memory = createMemory({
tokenLimit=40000,
shortTermTokenLimitRatio=0.5,
@@ -113,7 +113,7 @@ Currently, there are two predefined memory blocks:
This sounds a bit complicated, but it's actually quite simple. Let's look at an example:
```ts twoslash
```ts
import { createMemory, factExtractionBlock, staticBlock } from "llamaindex";
const memoryBlocks= [
@@ -142,7 +142,7 @@ You'll also notice that we've set the `priority` for the `factExtractionBlock` b
Now, let's pass these blocks into the `createMemory` function:
```ts twoslash
```ts
const memory = createMemory({
tokenLimit: 40000,
memoryBlocks: memoryBlocks,
@@ -11,6 +11,7 @@ A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a
- [KeywordTableLLMRetriever](/docs/api/classes/KeywordTableLLMRetriever) uses an LLM to extract keywords from the query and retrieve relevant nodes based on keyword matches.
- [KeywordTableSimpleRetriever](/docs/api/classes/KeywordTableSimpleRetriever) uses a basic frequency-based approach to extract keywords and retrieve nodes.
- [KeywordTableRAKERetriever](/docs/api/classes/KeywordTableRAKERetriever) uses the RAKE (Rapid Automatic Keyword Extraction) algorithm to extract keywords from the query, focusing on co-occurrence and context for keyword-based retrieval.
- [Bm25Retriever](/docs/api/classes/Bm25Retriever) uses the BM25 algorithm to extract keywords from the query and retrieve relevant nodes based on keyword matches.
```typescript
const retriever = vectorIndex.asRetriever({
@@ -1,5 +1,12 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.173
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.0.172
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.172",
"version": "0.0.173",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,11 @@
# @llamaindex/llama-parse-browser-test
## 0.0.72
### Patch Changes
- @llamaindex/cloud@4.0.17
## 0.0.71
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/llama-parse-browser-test",
"private": true,
"version": "0.0.71",
"version": "0.0.72",
"type": "module",
"scripts": {
"dev": "vite",
+7
View File
@@ -1,5 +1,12 @@
# @llamaindex/next-agent-test
## 0.1.173
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.1.172
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.172",
"version": "0.1.173",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,12 @@
# test-edge-runtime
## 0.1.172
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.1.171
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.171",
"version": "0.1.172",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,14 @@
# @llamaindex/next-node-runtime
## 0.1.41
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
- @llamaindex/huggingface@0.1.17
- @llamaindex/readers@3.1.12
## 0.1.40
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.1.40",
"version": "0.1.41",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,12 @@
# vite-import-llamaindex
## 0.0.39
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.0.38
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "vite-import-llamaindex",
"private": true,
"version": "0.0.38",
"version": "0.0.39",
"type": "module",
"scripts": {
"build": "vite build",
@@ -1,5 +1,12 @@
# @llamaindex/waku-query-engine-test
## 0.0.173
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.0.172
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.172",
"version": "0.0.173",
"type": "module",
"private": true,
"scripts": {
+56
View File
@@ -1,5 +1,61 @@
# examples
## 0.3.26
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- Updated dependencies [3cd8a57]
- Updated dependencies [f2dfd30]
- @llamaindex/core@0.6.13
- llamaindex@0.11.12
- @llamaindex/tools@0.1.3
- @llamaindex/bm25-retriever@0.0.2
- @llamaindex/cloud@4.0.17
- @llamaindex/node-parser@2.0.13
- @llamaindex/anthropic@0.3.15
- @llamaindex/assemblyai@0.1.12
- @llamaindex/clip@0.0.63
- @llamaindex/cohere@0.0.27
- @llamaindex/deepinfra@0.0.63
- @llamaindex/discord@0.1.12
- @llamaindex/google@0.3.12
- @llamaindex/huggingface@0.1.17
- @llamaindex/jinaai@0.0.23
- @llamaindex/mistral@0.1.13
- @llamaindex/mixedbread@0.0.27
- @llamaindex/notion@0.1.12
- @llamaindex/ollama@0.1.13
- @llamaindex/openai@0.4.7
- @llamaindex/perplexity@0.0.20
- @llamaindex/portkey-ai@0.0.55
- @llamaindex/replicate@0.0.55
- @llamaindex/astra@0.0.27
- @llamaindex/azure@0.1.24
- @llamaindex/chroma@0.0.27
- @llamaindex/elastic-search@0.1.13
- @llamaindex/firestore@1.0.20
- @llamaindex/milvus@0.1.22
- @llamaindex/mongodb@0.0.28
- @llamaindex/pinecone@0.1.13
- @llamaindex/postgres@0.0.56
- @llamaindex/qdrant@0.1.23
- @llamaindex/supabase@0.1.13
- @llamaindex/upstash@0.0.27
- @llamaindex/weaviate@0.0.28
- @llamaindex/vercel@0.1.13
- @llamaindex/voyage-ai@1.0.19
- @llamaindex/readers@3.1.12
- @llamaindex/workflow@1.1.13
- @llamaindex/deepseek@0.0.23
- @llamaindex/fireworks@0.0.23
- @llamaindex/groq@0.0.78
- @llamaindex/together@0.0.23
- @llamaindex/vllm@0.0.49
- @llamaindex/xai@0.0.10
## 0.3.25
### Patch Changes
+47 -46
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/examples",
"version": "0.3.25",
"version": "0.3.26",
"private": true,
"scripts": {
"lint": "eslint .",
@@ -11,51 +11,52 @@
"@azure/cosmos": "^4.1.1",
"@azure/identity": "^4.4.1",
"@azure/search-documents": "^12.1.0",
"@llamaindex/anthropic": "^0.3.14",
"@llamaindex/assemblyai": "^0.1.11",
"@llamaindex/astra": "^0.0.26",
"@llamaindex/azure": "^0.1.23",
"@llamaindex/chroma": "^0.0.26",
"@llamaindex/clip": "^0.0.62",
"@llamaindex/cloud": "^4.0.16",
"@llamaindex/cohere": "^0.0.26",
"@llamaindex/core": "^0.6.12",
"@llamaindex/deepinfra": "^0.0.62",
"@llamaindex/deepseek": "^0.0.22",
"@llamaindex/discord": "^0.1.11",
"@llamaindex/elastic-search": "^0.1.12",
"@llamaindex/anthropic": "^0.3.15",
"@llamaindex/assemblyai": "^0.1.12",
"@llamaindex/astra": "^0.0.27",
"@llamaindex/azure": "^0.1.24",
"@llamaindex/bm25-retriever": "^0.0.2",
"@llamaindex/chroma": "^0.0.27",
"@llamaindex/clip": "^0.0.63",
"@llamaindex/cloud": "^4.0.17",
"@llamaindex/cohere": "^0.0.27",
"@llamaindex/core": "^0.6.13",
"@llamaindex/deepinfra": "^0.0.63",
"@llamaindex/deepseek": "^0.0.23",
"@llamaindex/discord": "^0.1.12",
"@llamaindex/elastic-search": "^0.1.13",
"@llamaindex/env": "^0.1.30",
"@llamaindex/firestore": "^1.0.19",
"@llamaindex/fireworks": "^0.0.22",
"@llamaindex/google": "^0.3.11",
"@llamaindex/groq": "^0.0.77",
"@llamaindex/huggingface": "^0.1.16",
"@llamaindex/jinaai": "^0.0.22",
"@llamaindex/milvus": "^0.1.21",
"@llamaindex/mistral": "^0.1.12",
"@llamaindex/mixedbread": "^0.0.26",
"@llamaindex/mongodb": "^0.0.27",
"@llamaindex/node-parser": "^2.0.12",
"@llamaindex/notion": "^0.1.11",
"@llamaindex/ollama": "^0.1.12",
"@llamaindex/openai": "^0.4.6",
"@llamaindex/perplexity": "^0.0.19",
"@llamaindex/pinecone": "^0.1.12",
"@llamaindex/portkey-ai": "^0.0.54",
"@llamaindex/postgres": "^0.0.55",
"@llamaindex/qdrant": "^0.1.22",
"@llamaindex/readers": "^3.1.11",
"@llamaindex/replicate": "^0.0.54",
"@llamaindex/supabase": "^0.1.12",
"@llamaindex/together": "^0.0.22",
"@llamaindex/tools": "^0.1.1",
"@llamaindex/upstash": "^0.0.26",
"@llamaindex/vercel": "^0.1.12",
"@llamaindex/vllm": "^0.0.48",
"@llamaindex/voyage-ai": "^1.0.18",
"@llamaindex/weaviate": "^0.0.27",
"@llamaindex/workflow": "^1.1.12",
"@llamaindex/xai": "workspace:^0.0.9",
"@llamaindex/firestore": "^1.0.20",
"@llamaindex/fireworks": "^0.0.23",
"@llamaindex/google": "^0.3.12",
"@llamaindex/groq": "^0.0.78",
"@llamaindex/huggingface": "^0.1.17",
"@llamaindex/jinaai": "^0.0.23",
"@llamaindex/milvus": "^0.1.22",
"@llamaindex/mistral": "^0.1.13",
"@llamaindex/mixedbread": "^0.0.27",
"@llamaindex/mongodb": "^0.0.28",
"@llamaindex/node-parser": "^2.0.13",
"@llamaindex/notion": "^0.1.12",
"@llamaindex/ollama": "^0.1.13",
"@llamaindex/openai": "^0.4.7",
"@llamaindex/perplexity": "^0.0.20",
"@llamaindex/pinecone": "^0.1.13",
"@llamaindex/portkey-ai": "^0.0.55",
"@llamaindex/postgres": "^0.0.56",
"@llamaindex/qdrant": "^0.1.23",
"@llamaindex/readers": "^3.1.12",
"@llamaindex/replicate": "^0.0.55",
"@llamaindex/supabase": "^0.1.13",
"@llamaindex/together": "^0.0.23",
"@llamaindex/tools": "^0.1.3",
"@llamaindex/upstash": "^0.0.27",
"@llamaindex/vercel": "^0.1.13",
"@llamaindex/vllm": "^0.0.49",
"@llamaindex/voyage-ai": "^1.0.19",
"@llamaindex/weaviate": "^0.0.28",
"@llamaindex/workflow": "^1.1.13",
"@llamaindex/xai": "workspace:^0.0.10",
"@notionhq/client": "^2.2.15",
"@pinecone-database/pinecone": "^4.0.0",
"@vercel/postgres": "^0.10.0",
@@ -64,7 +65,7 @@
"commander": "^12.1.0",
"dotenv": "^16.4.5",
"js-tiktoken": "^1.0.14",
"llamaindex": "^0.11.11",
"llamaindex": "^0.11.12",
"mongodb": "6.7.0",
"postgres": "^3.4.4",
"wikipedia": "^2.1.2",
+1 -1
View File
@@ -1,4 +1,4 @@
import { LlamaParseReader } from "@llamaindex/cloud";
import { LlamaParseReader } from "@llamaindex/cloud/reader";
import { openai, OpenAIEmbedding } from "@llamaindex/openai";
import { Settings, VectorStoreIndex } from "llamaindex";
@@ -1,4 +1,4 @@
import { LlamaParseReader } from "@llamaindex/cloud";
import { LlamaParseReader } from "@llamaindex/cloud/reader";
import { SimpleDirectoryReader } from "@llamaindex/readers/directory";
import { VectorStoreIndex } from "llamaindex";
+14
View File
@@ -0,0 +1,14 @@
# BM25 Retriever
In this guide, we introduce a bm25 retriever that search documents using the bm25 method. BM25 (Best Matching 25) is a ranking function that extends TF-IDF by considering term frequency saturation and document length. BM25 effectively ranks documents based on query term occurrence and rarity across the corpus.
## Setup
1. `cd` Into the `examples` directory
2. run `npm i`
## Example
```bash
`npx tsx ./retrievers/bm25/example.ts`
```
+33
View File
@@ -0,0 +1,33 @@
import { Bm25Retriever } from "@llamaindex/bm25-retriever";
import { OpenAIEmbedding } from "@llamaindex/openai";
import { PDFReader } from "@llamaindex/readers/pdf";
import { MetadataMode, Settings, VectorStoreIndex } from "llamaindex";
Settings.embedModel = new OpenAIEmbedding();
async function main() {
// Load PDF
const reader = new PDFReader();
const documents = await reader.loadData("./data/brk-2022.pdf");
// Split text and create embeddings. Store them in a VectorStoreIndex
const index = await VectorStoreIndex.fromDocuments(documents);
const retriever = new Bm25Retriever({
docStore: index.docStore,
topK: 3,
});
// Query the data
const response = await retriever.retrieve({
query: "What mistakes did Warren E. Buffett make?",
});
// Output response
response.forEach((r) => {
console.log(`Score: ${r.score}`);
console.log(`Text: ${r.node.getContent(MetadataMode.NONE)}`);
});
}
main().catch(console.error);
+7
View File
@@ -1,5 +1,12 @@
# @llamaindex/autotool
## 8.0.12
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 8.0.11
### Patch Changes
@@ -1,5 +1,13 @@
# @llamaindex/autotool-01-node-example
## 0.0.120
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
- @llamaindex/autotool@8.0.12
## 0.0.119
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.119"
"version": "0.0.120"
}
+1 -1
View File
@@ -6,7 +6,7 @@
"url": "git+https://github.com/run-llama/LlamaIndexTS.git",
"directory": "packages/autotool"
},
"version": "8.0.11",
"version": "8.0.12",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/cloud
## 4.0.17
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 4.0.16
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloud",
"version": "4.0.16",
"version": "4.0.17",
"type": "module",
"license": "MIT",
"scripts": {
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/core
## 0.6.13
### Patch Changes
- d578889: Add new memory API
- 0fcc92f: Fix: split sentences must not trim whitespaces
- 515a8b9: Fix: logging for fromPersistPath
## 0.6.12
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/core",
"type": "module",
"version": "0.6.12",
"version": "0.6.13",
"description": "LlamaIndex Core Module",
"exports": {
"./agent": {
+7
View File
@@ -1,5 +1,12 @@
# @llamaindex/experimental
## 0.0.189
### Patch Changes
- Updated dependencies [515a8b9]
- llamaindex@0.11.12
## 0.0.188
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.188",
"version": "0.0.189",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+13
View File
@@ -1,5 +1,18 @@
# llamaindex
## 0.11.12
### Patch Changes
- 515a8b9: Fix: logging for fromPersistPath
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/cloud@4.0.17
- @llamaindex/node-parser@2.0.13
- @llamaindex/workflow@1.1.13
## 0.11.11
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.11.11",
"version": "0.11.12",
"license": "MIT",
"type": "module",
"keywords": [
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/core-test
## 0.1.8
### Patch Changes
- @llamaindex/openai@0.4.7
## 0.1.7
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/llamaindex-test",
"private": true,
"version": "0.1.7",
"version": "0.1.8",
"type": "module",
"scripts": {
"test": "vitest run"
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/node-parser
## 2.0.13
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 2.0.12
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/node-parser",
"version": "2.0.12",
"version": "2.0.13",
"description": "Node parser for LlamaIndex",
"type": "module",
"exports": {
@@ -1,5 +1,14 @@
# @llamaindex/anthropic
## 0.3.15
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.3.14
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/anthropic",
"description": "Anthropic Adapter for LlamaIndex",
"version": "0.3.14",
"version": "0.3.15",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,14 @@
# @llamaindex/assemblyai
## 0.1.12
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.11
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/assemblyai",
"description": "AssemblyAI Reader for LlamaIndex",
"version": "0.1.11",
"version": "0.1.12",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/community
## 0.0.108
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.107
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/aws",
"description": "AWS package for LlamaIndexTS",
"version": "0.0.107",
"version": "0.0.108",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+10
View File
@@ -1,5 +1,15 @@
# @llamaindex/clip
## 0.0.63
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.0.62
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/clip",
"description": "Clip Embedding Adapter for LlamaIndex",
"version": "0.0.62",
"version": "0.0.63",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/cohere
## 0.0.27
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.26
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/cohere",
"description": "Cohere Adapter for LlamaIndex",
"version": "0.0.26",
"version": "0.0.27",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+10
View File
@@ -1,5 +1,15 @@
# @llamaindex/deepinfra
## 0.0.63
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.0.62
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/deepinfra",
"description": "Deepinfra Adapter for LlamaIndex",
"version": "0.0.62",
"version": "0.0.63",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/deepseek
## 0.0.23
### Patch Changes
- @llamaindex/openai@0.4.7
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/deepseek",
"description": "DeepSeek Adapter for LlamaIndex",
"version": "0.0.22",
"version": "0.0.23",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/discord
## 0.1.12
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.11
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/discord",
"description": "Discord Reader for LlamaIndex",
"version": "0.1.11",
"version": "0.1.12",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/excel
## 0.1.13
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.12
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/excel",
"description": "Excel Reader for LlamaIndex",
"version": "0.1.12",
"version": "0.1.13",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
@@ -1,5 +1,11 @@
# @llamaindex/fireworks
## 0.0.23
### Patch Changes
- @llamaindex/openai@0.4.7
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/fireworks",
"description": "Fireworks Adapter for LlamaIndex",
"version": "0.0.22",
"version": "0.0.23",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/google
## 0.3.12
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.3.11
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/google",
"description": "Google Adapter for LlamaIndex",
"version": "0.3.11",
"version": "0.3.12",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/groq
## 0.0.78
### Patch Changes
- @llamaindex/openai@0.4.7
## 0.0.77
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/groq",
"description": "Groq Adapter for LlamaIndex",
"version": "0.0.77",
"version": "0.0.78",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,15 @@
# @llamaindex/huggingface
## 0.1.17
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.1.16
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/huggingface",
"description": "Huggingface Adapter for LlamaIndex",
"version": "0.1.16",
"version": "0.1.17",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
+10
View File
@@ -1,5 +1,15 @@
# @llamaindex/jinaai
## 0.0.23
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/jinaai",
"description": "JinaAI Adapter for LlamaIndex",
"version": "0.0.22",
"version": "0.0.23",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/mistral
## 0.1.13
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.12
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/mistral",
"description": "Mistral Adapter for LlamaIndex",
"version": "0.1.12",
"version": "0.1.13",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,14 @@
# @llamaindex/mixedbread
## 0.0.27
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.26
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/mixedbread",
"description": "Mixedbread Adapter for LlamaIndex",
"version": "0.0.26",
"version": "0.0.27",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/notion
## 0.1.12
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.11
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/notion",
"description": "Notion Reader for LlamaIndex",
"version": "0.1.11",
"version": "0.1.12",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/ollama
## 0.1.13
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.1.12
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/ollama",
"description": "Ollama Adapter for LlamaIndex",
"version": "0.1.12",
"version": "0.1.13",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
+9
View File
@@ -1,5 +1,14 @@
# @llamaindex/openai
## 0.4.7
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.4.6
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/openai",
"description": "OpenAI Adapter for LlamaIndex",
"version": "0.4.6",
"version": "0.4.7",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,15 @@
# @llamaindex/perplexity
## 0.0.20
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.0.19
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/perplexity",
"description": "Perplexity Adapter for LlamaIndex",
"version": "0.0.19",
"version": "0.0.20",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,14 @@
# @llamaindex/portkey-ai
## 0.0.55
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.54
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/portkey-ai",
"description": "Portkey Adapter for LlamaIndex",
"version": "0.0.54",
"version": "0.0.55",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,14 @@
# @llamaindex/replicate
## 0.0.55
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.54
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/replicate",
"description": "Replicate Adapter for LlamaIndex",
"version": "0.0.54",
"version": "0.0.55",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -0,0 +1,11 @@
# @llamaindex/bm25-retriever
## 0.0.2
### Patch Changes
- f2dfd30: feat: implement bm25 retriever
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
@@ -0,0 +1,52 @@
{
"name": "@llamaindex/bm25-retriever",
"description": "BM25 Retriever for LlamaIndex",
"version": "0.0.2",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"exports": {
".": {
"edge-light": {
"types": "./dist/index.edge-light.d.ts",
"default": "./dist/index.edge-light.js"
},
"workerd": {
"types": "./dist/index.edge-light.d.ts",
"default": "./dist/index.edge-light.js"
},
"require": {
"types": "./dist/index.d.cts",
"default": "./dist/index.cjs"
},
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
}
},
"files": [
"dist"
],
"repository": {
"type": "git",
"url": "git+https://github.com/run-llama/LlamaIndexTS.git",
"directory": "packages/providers/retriever/bm25"
},
"scripts": {
"build": "bunchee",
"dev": "bunchee --watch",
"test": "vitest"
},
"devDependencies": {
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"peerDependencies": {
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"dependencies": {
"okapibm25": "^1.4.1"
}
}
@@ -0,0 +1,58 @@
import type { QueryBundle } from "@llamaindex/core/query-engine";
import { BaseRetriever } from "@llamaindex/core/retriever";
import {
MetadataMode,
type BaseNode,
type NodeWithScore,
} from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store";
import { extractText } from "@llamaindex/core/utils";
import BM25 from "okapibm25";
export type Bm25RetrieverOptions = {
docStore: BaseDocumentStore;
topK?: number;
docIds?: string[];
};
export class Bm25Retriever extends BaseRetriever {
private docStore: BaseDocumentStore;
private topK: number;
private docIds: string[];
constructor(options: Bm25RetrieverOptions) {
super();
this.topK = options.topK || 10;
this.docStore = options.docStore;
this.docIds = options.docIds || [];
}
async _retrieve(params: QueryBundle): Promise<NodeWithScore[]> {
let nodes: BaseNode[] = [];
const { query } = params;
const queryStr = extractText(query);
if (this.docIds?.length) {
nodes = (
await Promise.all(
this.docIds.map((id) => this.docStore.getDocument(id, false)),
)
).filter((x) => !!x);
} else {
nodes = Object.values(await this.docStore.docs());
}
const contents = nodes.map(
(node) => node.getContent(MetadataMode.NONE) || "",
);
const scores = BM25(
contents,
queryStr.toLowerCase().split(/\s+/),
) as number[];
const scoredNodes = nodes.map((node, i) => ({
node,
score: scores[i] || 0,
}));
scoredNodes.sort((a, b) => b.score - a.score);
return scoredNodes.slice(0, this.topK);
}
}
@@ -0,0 +1 @@
export * from "./Bm25Retriever";
@@ -0,0 +1,204 @@
import { QueryBundle } from "@llamaindex/core/query-engine";
import { BaseNode, MetadataMode, TextNode } from "@llamaindex/core/schema";
import {
BaseDocumentStore,
RefDocInfo,
Serializer,
} from "@llamaindex/core/storage/doc-store";
import BM25 from "okapibm25";
import { beforeEach, describe, expect, test, vi } from "vitest";
import { Bm25Retriever } from "../src/Bm25Retriever";
// Mock okapibm25
vi.mock("okapibm25", () => {
return {
default: vi.fn((documents, queryTerms) => {
// Simple mock implementation that gives higher scores to documents containing query terms
return documents.map((doc) => {
const docTerms = doc.toLowerCase().split(/\s+/);
const matchCount = queryTerms.filter((term) =>
docTerms.includes(term),
).length;
return matchCount / queryTerms.length;
});
}),
};
});
// Mock document store
class MockDocumentStore implements BaseDocumentStore {
private documents: Record<string, BaseNode>;
constructor(documents: Record<string, BaseNode> = {}) {
this.documents = documents;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
serializer: Serializer<any>;
persist(persistPath?: string): void {
throw new Error("Method not implemented.");
}
documentExists(docId: string): Promise<boolean> {
throw new Error("Method not implemented.");
}
setDocumentHash(docId: string, docHash: string): Promise<void> {
throw new Error("Method not implemented.");
}
getDocumentHash(docId: string): Promise<string | undefined> {
throw new Error("Method not implemented.");
}
getAllDocumentHashes(): Promise<Record<string, string>> {
throw new Error("Method not implemented.");
}
getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined> {
throw new Error("Method not implemented.");
}
getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined> {
throw new Error("Method not implemented.");
}
deleteRefDoc(refDocId: string, raiseError: boolean): Promise<void> {
throw new Error("Method not implemented.");
}
getNodes(nodeIds: string[], raiseError?: boolean): Promise<BaseNode[]> {
throw new Error("Method not implemented.");
}
getNode(nodeId: string, raiseError?: boolean): Promise<BaseNode> {
throw new Error("Method not implemented.");
}
getNodeDict(nodeIdDict: {
[index: number]: string;
}): Promise<Record<number, BaseNode>> {
throw new Error("Method not implemented.");
}
deleteDocument(docId: string, raiseError = true): Promise<void> {
throw new Error("Method not implemented.");
}
async getDocument(docId: string, raiseError = true): Promise<BaseNode> {
const doc = this.documents[docId];
if (!doc && raiseError) {
throw new Error(`Document with ID ${docId} not found`);
}
return doc;
}
async addDocuments(docs: BaseNode[], allowUpdate = true): Promise<void> {
for (const doc of docs) {
this.documents[doc.id_] = doc;
}
}
async docs(): Promise<Record<string, BaseNode>> {
return { ...this.documents };
}
}
describe("Bm25Retriever", () => {
let mockDocStore: MockDocumentStore;
let nodes: BaseNode[];
beforeEach(() => {
nodes = [
new TextNode({ text: "The quick brown fox jumps over the lazy dog" }),
new TextNode({ text: "A quick brown dog outpaces the swift fox" }),
new TextNode({ text: "Lazy cats sleep all day long" }),
new TextNode({ text: "Brown foxes are known for their intelligence" }),
new TextNode({ text: "Dogs are man's best friend" }),
];
mockDocStore = new MockDocumentStore();
mockDocStore.addDocuments(nodes);
});
test("should initialize with default values", () => {
const retriever = new Bm25Retriever({ docStore: mockDocStore });
expect(retriever).toBeInstanceOf(Bm25Retriever);
});
test("should initialize with custom values", () => {
const retriever = new Bm25Retriever({
docStore: mockDocStore,
topK: 5,
docIds: ["node1", "node2"],
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((retriever as any).topK).toBe(5);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((retriever as any).docIds).toEqual(["node1", "node2"]);
});
test("should retrieve documents with BM25 ranking", async () => {
const retriever = new Bm25Retriever({ docStore: mockDocStore });
const query: QueryBundle = { query: "quick fox" };
const results = await retriever._retrieve(query);
expect(BM25).toHaveBeenCalled();
expect(results.length).toBeGreaterThan(0);
for (let i = 0; i < results.length - 1; i++) {
expect(results[i].score).toBeDefined();
expect(results[i].score).toBeGreaterThanOrEqual(
results[i + 1].score || 0,
);
}
const topResult = results[0];
expect(
topResult.node.getContent(MetadataMode.NONE).toLowerCase(),
).toContain("quick");
expect(
topResult.node.getContent(MetadataMode.NONE).toLowerCase(),
).toContain("fox");
});
test("should respect topK parameter", async () => {
const topK = 2;
const retriever = new Bm25Retriever({
docStore: mockDocStore,
topK,
});
const query: QueryBundle = { query: "brown" };
const results = await retriever._retrieve(query);
expect(results.length).toBe(topK);
});
test("should filter by docIds if provided", async () => {
const docIds = [nodes[0].id_, nodes[1].id_];
const retriever = new Bm25Retriever({
docStore: mockDocStore,
docIds,
});
const query: QueryBundle = { query: "lazy" };
const results = await retriever._retrieve(query);
const returnedIds = results.map((result) => result.node.id_);
expect(returnedIds.every((id) => docIds.includes(id))).toBe(true);
expect(returnedIds).toContain(nodes[0].id_);
expect(returnedIds).toContain(nodes[1].id_);
});
test("should handle empty query", async () => {
const retriever = new Bm25Retriever({ docStore: mockDocStore });
const query: QueryBundle = { query: "" };
const results = await retriever._retrieve(query);
expect(results.length).toBeGreaterThan(0);
});
test("should handle non-existent docIds", async () => {
const docIds = [nodes[0].id_, "nonexistent"];
const retriever = new Bm25Retriever({
docStore: mockDocStore,
docIds,
});
const query: QueryBundle = { query: "fox" };
const results = await retriever._retrieve(query);
const returnedIds = results.map((result) => result.node.id_);
expect(returnedIds).toContain(nodes[0].id_);
expect(returnedIds).not.toContain("nonexistent");
});
});
@@ -0,0 +1,19 @@
{
"extends": "../../../../tsconfig.json",
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"outDir": "./lib",
"tsBuildInfoFile": "./lib/.tsbuildinfo"
},
"include": ["./src"],
"references": [
{
"path": "../../../core/tsconfig.json"
},
{
"path": "../../../env/tsconfig.json"
}
]
}
@@ -1,5 +1,14 @@
# @llamaindex/astra
## 0.0.27
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
## 0.0.26
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/astra",
"description": "Astra Storage for LlamaIndex",
"version": "0.0.26",
"version": "0.0.27",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,5 +1,15 @@
# @llamaindex/azure
## 0.1.24
### Patch Changes
- Updated dependencies [d578889]
- Updated dependencies [0fcc92f]
- Updated dependencies [515a8b9]
- @llamaindex/core@0.6.13
- @llamaindex/openai@0.4.7
## 0.1.23
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/azure",
"description": "Azure Storage for LlamaIndex",
"version": "0.1.23",
"version": "0.1.24",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",

Some files were not shown because too many files have changed in this diff Show More