Compare commits

..

3 Commits

Author SHA1 Message Date
Alex Yang e921134f7f Create strong-mice-dance.md 2024-11-21 23:05:33 -08:00
Alex Yang 6d6dcff02f docs: update 2024-11-21 22:58:29 -08:00
Alex Yang 679e3325c6 feat: add cloudflare worker full example 2024-11-21 21:22:39 -08:00
71 changed files with 37 additions and 1215 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"@llamaindex/env": patch
---
fix: async local storage on cloudflare worker
-20
View File
@@ -1,25 +1,5 @@
# docs
## 0.0.129
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.128
### Patch Changes
- llamaindex@0.8.23
## 0.0.127
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.0.126
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "docs",
"version": "0.0.129",
"version": "0.0.126",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
-30
View File
@@ -1,35 +1,5 @@
# @llamaindex/doc
## 0.0.27
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.26
### Patch Changes
- @llamaindex/cloud@2.0.15
- @llamaindex/core@0.4.15
- llamaindex@0.8.23
- @llamaindex/node-parser@0.0.16
- @llamaindex/openai@0.1.40
- @llamaindex/readers@1.0.17
## 0.0.25
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
- @llamaindex/cloud@2.0.14
- @llamaindex/core@0.4.14
- @llamaindex/node-parser@0.0.15
- @llamaindex/openai@0.1.39
- @llamaindex/readers@1.0.16
## 0.0.24
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.0.27",
"version": "0.0.24",
"private": true,
"scripts": {
"build": "pnpm run build:docs && next build",
@@ -1,25 +1,5 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.120
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.119
### Patch Changes
- llamaindex@0.8.23
## 0.0.118
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.0.117
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.120",
"version": "0.0.117",
"type": "module",
"private": true,
"scripts": {
@@ -1,17 +1,5 @@
# @llamaindex/llama-parse-browser-test
## 0.0.35
### Patch Changes
- @llamaindex/cloud@2.0.15
## 0.0.34
### Patch Changes
- @llamaindex/cloud@2.0.14
## 0.0.33
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/llama-parse-browser-test",
"private": true,
"version": "0.0.35",
"version": "0.0.33",
"type": "module",
"scripts": {
"dev": "vite",
-20
View File
@@ -1,25 +1,5 @@
# @llamaindex/next-agent-test
## 0.1.120
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.1.119
### Patch Changes
- llamaindex@0.8.23
## 0.1.118
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.1.117
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.120",
"version": "0.1.117",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,25 +1,5 @@
# test-edge-runtime
## 0.1.119
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.1.118
### Patch Changes
- llamaindex@0.8.23
## 0.1.117
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.1.116
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.119",
"version": "0.1.116",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,25 +1,5 @@
# @llamaindex/next-node-runtime
## 0.0.101
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.100
### Patch Changes
- llamaindex@0.8.23
## 0.0.99
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.0.98
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.0.101",
"version": "0.0.98",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,25 +1,5 @@
# @llamaindex/waku-query-engine-test
## 0.0.120
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.119
### Patch Changes
- llamaindex@0.8.23
## 0.0.118
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.0.117
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.120",
"version": "0.0.117",
"type": "module",
"private": true,
"scripts": {
+1 -2
View File
@@ -15,8 +15,7 @@
"start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
"start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
"start:discord": "node --import tsx ./src/discord.ts",
"start:json": "node --import tsx ./src/json.ts",
"start:obsidian": "node --import tsx ./src/obsidian.ts"
"start:json": "node --import tsx ./src/json.ts"
},
"dependencies": {
"@llamaindex/readers": "*",
-12
View File
@@ -1,12 +0,0 @@
import { ObsidianReader } from "@llamaindex/readers/obsidian";
const obsidianReader = new ObsidianReader(
"/Users/jingyi/Documents/jingyi-vault",
);
obsidianReader.loadData().then((documents) => {
console.log("documents:", documents.length);
documents.forEach((doc) => {
console.log(`document (${doc.id_}):`, doc.getText());
});
});
-20
View File
@@ -1,25 +1,5 @@
# @llamaindex/autotool
## 5.0.24
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 5.0.23
### Patch Changes
- llamaindex@0.8.23
## 5.0.22
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 5.0.21
### Patch Changes
@@ -1,28 +1,5 @@
# @llamaindex/autotool-01-node-example
## 0.0.67
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
- @llamaindex/autotool@5.0.24
## 0.0.66
### Patch Changes
- llamaindex@0.8.23
- @llamaindex/autotool@5.0.23
## 0.0.65
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
- @llamaindex/autotool@5.0.22
## 0.0.64
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.67"
"version": "0.0.64"
}
@@ -1,28 +1,5 @@
# @llamaindex/autotool-02-next-example
## 0.1.111
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
- @llamaindex/autotool@5.0.24
## 0.1.110
### Patch Changes
- llamaindex@0.8.23
- @llamaindex/autotool@5.0.23
## 0.1.109
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
- @llamaindex/autotool@5.0.22
## 0.1.108
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool-02-next-example",
"private": true,
"version": "0.1.111",
"version": "0.1.108",
"scripts": {
"dev": "next dev",
"build": "next build",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool",
"type": "module",
"version": "5.0.24",
"version": "5.0.21",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/cloud
## 2.0.15
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 2.0.14
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 2.0.13
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloud",
"version": "2.0.15",
"version": "2.0.13",
"type": "module",
"license": "MIT",
"scripts": {
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/community
## 0.0.73
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.72
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.71
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/community",
"description": "Community package for LlamaIndexTS",
"version": "0.0.73",
"version": "0.0.71",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
-14
View File
@@ -1,19 +1,5 @@
# @llamaindex/core
## 0.4.15
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
## 0.4.14
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
## 0.4.13
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/core",
"type": "module",
"version": "0.4.15",
"version": "0.4.13",
"description": "LlamaIndex Core Module",
"exports": {
"./agent": {
-12
View File
@@ -1,17 +1,5 @@
# @llamaindex/env
## 0.1.23
### Patch Changes
- d2b2722: fix: switch tokenizer in cloudflare
## 0.1.22
### Patch Changes
- 969365c: fix: async local storage on cloudflare worker
## 0.1.21
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/env",
"description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
"version": "0.1.23",
"version": "0.1.21",
"type": "module",
"types": "dist/index.d.ts",
"module": "dist/index.js",
+1 -1
View File
@@ -2,4 +2,4 @@ export {
Tokenizers,
tokenizers,
type Tokenizer,
} from "./internal/tokenizers/node.js";
} from "./internal/tokenizers/js.js";
-20
View File
@@ -1,25 +1,5 @@
# @llamaindex/experimental
## 0.0.136
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.135
### Patch Changes
- llamaindex@0.8.23
## 0.0.134
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
## 0.0.133
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.136",
"version": "0.0.133",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
-49
View File
@@ -1,54 +1,5 @@
# llamaindex
## 0.8.24
### Patch Changes
- 515f2c1: Add vector store for CosmosDB
## 0.8.23
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/cloud@2.0.15
- @llamaindex/core@0.4.15
- @llamaindex/node-parser@0.0.16
- @llamaindex/anthropic@0.0.24
- @llamaindex/clip@0.0.24
- @llamaindex/deepinfra@0.0.24
- @llamaindex/groq@0.0.39
- @llamaindex/huggingface@0.0.24
- @llamaindex/ollama@0.0.31
- @llamaindex/openai@0.1.40
- @llamaindex/portkey-ai@0.0.24
- @llamaindex/replicate@0.0.24
- @llamaindex/readers@1.0.17
- @llamaindex/vllm@0.0.10
## 0.8.22
### Patch Changes
- 819af45: feat: add Azure Cosmos DB Mongo vCore ChatStore
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/cloud@2.0.14
- @llamaindex/core@0.4.14
- @llamaindex/node-parser@0.0.15
- @llamaindex/anthropic@0.0.23
- @llamaindex/clip@0.0.23
- @llamaindex/deepinfra@0.0.23
- @llamaindex/groq@0.0.38
- @llamaindex/huggingface@0.0.23
- @llamaindex/ollama@0.0.30
- @llamaindex/openai@0.1.39
- @llamaindex/portkey-ai@0.0.23
- @llamaindex/replicate@0.0.23
- @llamaindex/readers@1.0.16
- @llamaindex/vllm@0.0.9
## 0.8.21
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.8.24",
"version": "0.8.21",
"license": "MIT",
"type": "module",
"keywords": [
-1
View File
@@ -15,6 +15,5 @@ export * from "@llamaindex/readers/json";
export * from "@llamaindex/readers/markdown";
export * from "@llamaindex/readers/mongo";
export * from "@llamaindex/readers/notion";
export * from "@llamaindex/readers/obsidian";
export * from "@llamaindex/readers/pdf";
export * from "@llamaindex/readers/text";
@@ -1,186 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import type {
ChatMessage,
MessageContent,
MessageType,
} from "@llamaindex/core/llms";
import { BaseChatStore } from "@llamaindex/core/storage/chat-store";
import type { Collection } from "mongodb";
import { MongoClient } from "mongodb";
const DEFAULT_CHAT_DATABASE = "ChatStoreDB";
const DEFAULT_CHAT_Collection = "ChatStoreCollection";
export interface AzureCosmosVCoreChatStoreConfig {
mongoClient?: MongoClient;
dbName?: string;
collectionName?: string;
}
export class AzureCosmosVCoreChatStore<
AdditionalMessageOptions extends object = object,
> extends BaseChatStore<AdditionalMessageOptions> {
private mongoClient: MongoClient;
private dbName: string;
private collectionName: string;
private collection?: Collection;
/**
* Create a new AzureCosmosVCoreChatStore instance.
*/
constructor({
mongoClient,
dbName = DEFAULT_CHAT_DATABASE,
collectionName = DEFAULT_CHAT_Collection,
}: AzureCosmosVCoreChatStoreConfig) {
super();
if (!mongoClient) {
throw new Error(
"MongoClient is required for AzureCosmosVCoreChatStore initialization",
);
}
this.mongoClient = mongoClient;
this.dbName = dbName;
this.collectionName = collectionName;
}
static fromMongoClient(
mongoClient: MongoClient,
dbName: string = DEFAULT_CHAT_DATABASE,
collectionName: string = DEFAULT_CHAT_Collection,
) {
return new AzureCosmosVCoreChatStore({
mongoClient,
dbName,
collectionName,
});
}
client(): MongoClient {
return this.mongoClient;
}
private convertToChatMessage(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
message: any,
): ChatMessage<AdditionalMessageOptions> {
return {
content: message.content as MessageContent,
role: message.role as MessageType,
options: message.options as AdditionalMessageOptions,
} as ChatMessage<AdditionalMessageOptions>;
}
private convertTovCoreMessage(
message: ChatMessage<AdditionalMessageOptions>,
): // eslint-disable-next-line @typescript-eslint/no-explicit-any
any {
return {
content: message.content,
role: message.role,
options: message.options,
};
}
private async ensureCollection(): Promise<Collection> {
if (!this.collection) {
this.collection = this.mongoClient
.db(this.dbName)
.collection(this.collectionName);
}
return this.collection;
}
/**
* Set messages for a given key.
*/
async setMessages(
key: string,
messages: ChatMessage<AdditionalMessageOptions>[],
): Promise<void> {
const collection = await this.ensureCollection();
const inputMessages = messages.map(this.convertTovCoreMessage);
await collection.updateOne(
{ id: key },
{ $set: { messages: inputMessages } },
{ upsert: true },
);
}
/**
* Get messages for a given key.
*/
async getMessages(
key: string,
): Promise<ChatMessage<AdditionalMessageOptions>[]> {
const collection = await this.ensureCollection();
const res = await collection.findOne({
id: key,
});
const messageHistory = res?.messages ?? [];
const result = messageHistory.map(this.convertToChatMessage);
return result;
}
/**
* Add a message for a given key.
*/
async addMessage(
key: string,
message: ChatMessage<AdditionalMessageOptions>,
idx?: number,
): Promise<void> {
const collection = await this.ensureCollection();
const res = await this.getMessages(key);
const messageHistory = res.map(this.convertTovCoreMessage) ?? [];
messageHistory.splice(
idx ?? messageHistory.length,
0,
this.convertTovCoreMessage(message),
);
await collection.updateOne(
{ id: key },
{ $set: { messages: messageHistory } },
{ upsert: true },
);
}
/**
* Deletes all messages for a given key.
*/
async deleteMessages(key: string): Promise<void> {
const collection = await this.ensureCollection();
try {
await collection.deleteOne({ id: key });
// eslint-disable-next-line no-empty
} catch (e) {}
}
/**
* Deletes one message at idx index for a given key.
*/
async deleteMessage(key: string, idx: number): Promise<void> {
// const collection = await this.ensureCollection();
const messageHistory = await this.getMessages(key);
if (idx >= 0 && idx < messageHistory.length) {
messageHistory.splice(idx, 1);
await this.setMessages(key, messageHistory);
}
}
/**
* Get all keys.
*/
async getKeys(): Promise<IterableIterator<string>> {
const collection = await this.ensureCollection();
const keys = await collection.distinct("id");
function* keyGenerator(): IterableIterator<string> {
for (const key of keys) {
yield key;
}
}
return keyGenerator();
}
}
-1
View File
@@ -2,7 +2,6 @@ export * from "@llamaindex/core/storage/chat-store";
export * from "@llamaindex/core/storage/doc-store";
export * from "@llamaindex/core/storage/index-store";
export * from "@llamaindex/core/storage/kv-store";
export * from "./chatStore/AzureCosmosMongovCoreChatStore.js";
export * from "./chatStore/AzureCosmosNoSqlChatStore.js";
export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js";
export * from "./docStore/AzureCosmosNoSqlDocumentStore.js";
@@ -1,328 +0,0 @@
import type { BaseNode } from "@llamaindex/core/schema";
import { MetadataMode } from "@llamaindex/core/schema";
import { getEnv } from "@llamaindex/env";
import { Collection, Db, MongoClient } from "mongodb";
import {
BaseVectorStore,
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
} from "./types.js";
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
/** Azure Cosmos DB for MongoDB vCore Similarity type. */
export const AzureCosmosDBMongoDBSimilarityType = {
/** Cosine similarity */
COS: "COS",
/** Inner - product */
IP: "IP",
/** Euclidian distance */
L2: "L2",
} as const;
/** Azure Cosmos DB for MongoDB vCore Similarity type. */
export type AzureCosmosDBMongoDBSimilarityType =
(typeof AzureCosmosDBMongoDBSimilarityType)[keyof typeof AzureCosmosDBMongoDBSimilarityType];
/** Azure Cosmos DB for MongoDB vCore Index Options. */
export type AzureCosmosDBMongoDBIndexOptions = {
readonly indexType?: "ivf" | "hnsw" | "diskann" | undefined;
/** Number of clusters that the inverted file (IVF) index uses to group the vector data. */
readonly numLists?: number | undefined;
/** Number of dimensions for vector similarity. */
readonly dimensions?: number | undefined;
/** Similarity metric to use with the IVF index. */
readonly similarity?: AzureCosmosDBMongoDBSimilarityType | undefined;
/** The max number of connections per layer with the HNSW index. */
readonly m?: number | undefined;
/** The size of the dynamic candidate list for constructing the graph with the HNSW index. */
readonly efConstruction?: number | undefined;
/** Max number of neighbors withe the Diskann idnex */
readonly maxDegree?: number | undefined;
/** L value for index building withe the Diskann idnex */
readonly lBuild?: number | undefined;
/** L value for index searching withe the Diskann idnex */
readonly lSearch?: number | undefined;
};
/**
* Azure Cosmos DB for MongoDB vCore vector store.
* To use this, you should have both:
* - the `mongodb` NPM package installed
* - a connection string associated with a MongoDB VCore Cluster
*
* You do not need to create a database or collection, it will be created
* automatically.
*
* You also need an index on the collection, which is by default be created
* automatically using the `createIndex` method.
*/
export class AzureCosmosDBMongoDBVectorStore extends BaseVectorStore {
storesText: boolean = true;
flatMetadata: boolean = true;
dbName: string;
collectionName: string;
indexedMetadataFields: string[];
/**
* The used MongoClient. If not given, a new MongoClient is created based on the MONGODB_URI env variable.
*/
mongodbClient: MongoClient;
indexName: string;
embeddingKey: string;
idKey: string;
textKey: string;
metadataKey: string;
indexOptions: AzureCosmosDBMongoDBIndexOptions;
private collection?: Collection;
private database: Db;
constructor(
init: Partial<AzureCosmosDBMongoDBVectorStore> & {
dbName: string;
collectionName: string;
indexedMetadataFields?: string[];
} & VectorStoreBaseParams,
) {
super(init);
if (init.mongodbClient) {
this.mongodbClient = init.mongodbClient;
} else {
const mongoUri = getEnv("AZURE_COSMOSDB_MONGODB_CONNECTION_STRING");
if (!mongoUri) {
throw new Error(
"AzureCosmosDBMongoDBVectorStore client or connection string must be set.",
);
}
this.mongodbClient = new MongoClient(mongoUri);
}
this.dbName = init.dbName ?? "documentsDB";
this.collectionName = init.collectionName ?? "documents";
this.indexedMetadataFields = init.indexedMetadataFields ?? [];
this.indexName = init.indexName ?? "vectorSearchIndex";
this.embeddingKey = init.embeddingKey ?? "vectorContent";
this.idKey = init.idKey ?? "id";
this.textKey = init.textKey ?? "text";
this.metadataKey = init.metadataKey ?? "metadata";
this.indexOptions = init.indexOptions ?? {};
this.database = this.mongodbClient.db(this.dbName);
}
client() {
return this.mongodbClient;
}
async ensureCollection() {
if (!this.collection) {
const collection = await this.mongodbClient
.db(this.dbName)
.createCollection(this.collectionName);
this.collection = collection;
}
return this.collection;
}
async add(nodes: BaseNode[]): Promise<string[]> {
if (!nodes || nodes.length === 0) {
return [];
}
const dataToInsert = nodes.map((node) => {
const metadata = nodeToMetadata(
node,
true,
this.textKey,
this.flatMetadata,
);
// Include the specified metadata fields in the top level of the document (to help filter)
const populatedMetadata: Record<string, unknown> = {};
for (const field of this.indexedMetadataFields) {
populatedMetadata[field] = metadata[field];
}
return {
[this.idKey]: node.id_,
[this.embeddingKey]: node.getEmbedding(),
[this.textKey]: node.getContent(MetadataMode.NONE) || "",
[this.metadataKey]: metadata,
...populatedMetadata,
};
});
const collection = await this.ensureCollection();
const insertResult = await collection.insertMany(dataToInsert);
return Object.values(insertResult.insertedIds).map((id) => String(id));
}
/**
* Removes specified documents from the AzureCosmosDBMongoDBVectorStore.
* @param params Parameters for the delete operation.
* @returns A promise that resolves when the documents have been removed.
*/
async delete(id: string, deleteOptions?: object): Promise<void> {
const collection = await this.ensureCollection();
await collection.deleteMany(
{
id: id,
},
deleteOptions,
);
}
async query(
query: VectorStoreQuery,
options?: object,
): Promise<VectorStoreQueryResult> {
const pipeline = [
{
$search: {
cosmosSearch: {
vector: query.queryEmbedding,
path: this.embeddingKey,
k: query.similarityTopK ?? 4,
},
returnStoredSource: true,
},
},
];
const collection = await this.ensureCollection();
const cursor = await collection.aggregate(pipeline);
const nodes: BaseNode[] = [];
const ids: string[] = [];
const similarities: number[] = [];
for await (const res of await cursor) {
const text = res[this.textKey];
const score = res.score;
const id = res[this.idKey];
const metadata = res[this.metadataKey];
const node = metadataDictToNode(metadata);
node.setContent(text);
ids.push(id);
nodes.push(node);
similarities.push(score);
}
const result = {
nodes,
similarities,
ids,
};
return result;
}
/**
* Creates an index on the collection with the specified index name during
* instance construction.
*
* Setting the numLists parameter correctly is important for achieving good
* accuracy and performance.
* Since the vector store uses IVF as the indexing strategy, you should
* create the index only after you have loaded a large enough sample
* documents to ensure that the centroids for the respective buckets are
* faily distributed.
*
* @param indexType Index Type for Mongo vCore index.
* @param dimensions Number of dimensions for vector similarity.
* The maximum number of supported dimensions is 2000.
* If no number is provided, it will be determined automatically by
* embedding a short text.
* @param similarity Similarity metric to use with the IVF index.
* Possible options are:
* - CosmosDBSimilarityType.COS (cosine distance)
* - CosmosDBSimilarityType.L2 (Euclidean distance)
* - CosmosDBSimilarityType.IP (inner product)
* @returns A promise that resolves when the index has been created.
*/
async createIndex(
dimensions: number | undefined = undefined,
indexType: "ivf" | "hnsw" | "diskann" = "ivf",
similarity: AzureCosmosDBMongoDBSimilarityType = AzureCosmosDBMongoDBSimilarityType.COS,
): Promise<void> {
let vectorLength = dimensions;
if (vectorLength === undefined) {
vectorLength = 1536;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cosmosSearchOptions: any = {
kind: "",
similarity,
dimensions: vectorLength,
};
if (indexType === "hnsw") {
cosmosSearchOptions.kind = "vector-hnsw";
cosmosSearchOptions.m = this.indexOptions.m ?? 16;
cosmosSearchOptions.efConstruction =
this.indexOptions.efConstruction ?? 200;
} else if (indexType === "diskann") {
cosmosSearchOptions.kind = "vector-diskann";
cosmosSearchOptions.maxDegree = this.indexOptions.maxDegree ?? 40;
cosmosSearchOptions.lBuild = this.indexOptions.lBuild ?? 50;
cosmosSearchOptions.lSearch = this.indexOptions.lSearch ?? 40;
/** Default to IVF index */
} else {
cosmosSearchOptions.kind = "vector-ivf";
cosmosSearchOptions.numLists = this.indexOptions.numLists ?? 100;
}
const createIndexCommands = {
createIndexes: this.collection?.collectionName,
indexes: [
{
name: this.indexName,
key: { [this.embeddingKey]: "cosmosSearch" },
cosmosSearchOptions,
},
],
};
await this.database.command(createIndexCommands);
}
/**
* Checks if the specified index name during instance construction exists
* on the collection.
* @returns A promise that resolves to a boolean indicating if the index exists.
*/
async checkIndexExists(): Promise<boolean> {
const collection = await this.ensureCollection();
const indexes = await collection.listIndexes().toArray();
return indexes.some((index) => index.name === this.indexName);
}
/**
* Deletes the index specified during instance construction if it exists.
* @returns A promise that resolves when the index has been deleted.
*/
async deleteIndex(indexName: string): Promise<void> {
const collection = await this.ensureCollection();
const indexes = await collection.listIndexes().toArray();
const indexToDelete = indexes.find((index) => index.name === indexName);
if (indexToDelete) {
await collection.dropIndex(indexName);
}
}
}
@@ -1,5 +1,4 @@
export * from "./AstraDBVectorStore.js";
export * from "./AzureCosmosDBMongoVectorStore.js";
export * from "./AzureCosmosDBNoSqlVectorStore.js";
export * from "./ChromaVectorStore.js";
export * from "./MilvusVectorStore.js";
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/node-parser
## 0.0.16
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.15
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.14
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/node-parser",
"version": "0.0.16",
"version": "0.0.14",
"description": "Node parser for LlamaIndex",
"type": "module",
"exports": {
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/anthropic
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/anthropic",
"description": "Anthropic Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-18
View File
@@ -1,23 +1,5 @@
# @llamaindex/clip
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
- @llamaindex/openai@0.1.40
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
- @llamaindex/openai@0.1.39
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/clip",
"description": "Clip Embedding Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
-18
View File
@@ -1,23 +1,5 @@
# @llamaindex/deepinfra
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
- @llamaindex/openai@0.1.40
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
- @llamaindex/openai@0.1.39
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/deepinfra",
"description": "Deepinfra Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/groq
## 0.0.39
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/openai@0.1.40
## 0.0.38
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/openai@0.1.39
## 0.0.37
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/groq",
"description": "Groq Adapter for LlamaIndex",
"version": "0.0.39",
"version": "0.0.37",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,23 +1,5 @@
# @llamaindex/huggingface
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
- @llamaindex/openai@0.1.40
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
- @llamaindex/openai@0.1.39
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/huggingface",
"description": "Huggingface Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"types": "dist/index.d.ts",
"main": "dist/index.cjs",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/ollama
## 0.0.31
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.30
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.29
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/ollama",
"description": "Ollama Adapter for LlamaIndex",
"version": "0.0.31",
"version": "0.0.29",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/openai
## 0.1.40
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.1.39
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.1.38
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/openai",
"description": "OpenAI Adapter for LlamaIndex",
"version": "0.1.40",
"version": "0.1.38",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
@@ -1,21 +1,5 @@
# @llamaindex/portkey-ai
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/portkey-ai",
"description": "Portkey Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/replicate
## 0.0.24
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 0.0.23
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/replicate",
"description": "Replicate Adapter for LlamaIndex",
"version": "0.0.24",
"version": "0.0.22",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-12
View File
@@ -1,17 +1,5 @@
# @llamaindex/vllm
## 0.0.10
### Patch Changes
- @llamaindex/openai@0.1.40
## 0.0.9
### Patch Changes
- @llamaindex/openai@0.1.39
## 0.0.8
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/vllm",
"description": "vLLM Adapter for LlamaIndex",
"version": "0.0.10",
"version": "0.0.8",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
-16
View File
@@ -1,21 +1,5 @@
# @llamaindex/readers
## 1.0.17
### Patch Changes
- Updated dependencies [d2b2722]
- @llamaindex/env@0.1.23
- @llamaindex/core@0.4.15
## 1.0.16
### Patch Changes
- Updated dependencies [969365c]
- @llamaindex/env@0.1.22
- @llamaindex/core@0.4.14
## 1.0.15
### Patch Changes
-14
View File
@@ -1,14 +0,0 @@
{
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"edge-light": "./dist/index.edge-light.js",
"workerd": "./dist/index.workerd.js",
"default": "./dist/index.js"
}
},
"private": true
}
+1 -20
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/readers",
"description": "LlamaIndex Readers",
"version": "1.0.17",
"version": "1.0.15",
"type": "module",
"exports": {
"./node/hook": "./node/dist/hook.js",
@@ -142,24 +142,6 @@
"default": "./notion/dist/index.js"
}
},
"./obsidian": {
"edge-light": {
"types": "./obsidian/dist/index.edge-light.d.ts",
"default": "./obsidian/dist/index.edge-light.js"
},
"workerd": {
"types": "./obsidian/dist/index.workerd.d.ts",
"default": "./obsidian/dist/index.workerd.js"
},
"require": {
"types": "./obsidian/dist/index.d.cts",
"default": "./obsidian/dist/index.cjs"
},
"import": {
"types": "./obsidian/dist/index.d.ts",
"default": "./obsidian/dist/index.js"
}
},
"./pdf": {
"require": {
"types": "./pdf/dist/index.d.cts",
@@ -194,7 +176,6 @@
"markdown",
"mongo",
"notion",
"obsidian",
"pdf",
"text",
"node"
-41
View File
@@ -1,41 +0,0 @@
import { type BaseReader, Document } from "@llamaindex/core/schema";
import * as fs from "node:fs";
import path from "node:path";
import { MarkdownReader } from "./markdown";
export class ObsidianReader implements BaseReader<Document> {
protected inputDir: string;
protected docs: Document[] = [];
constructor(inputDir: string) {
this.inputDir = inputDir;
}
private async processPath(file: fs.Dirent, filepath: string) {
if (file.isDirectory() && !file.name.startsWith(".")) {
await this.readFromPath(filepath);
} else if (file.isFile() && file.name.endsWith(".md")) {
await this.convertToDocuments(filepath);
} else {
console.log(`Skipping ${filepath}`);
}
}
private async readFromPath(dir: string) {
const files = await fs.promises.readdir(dir, { withFileTypes: true });
for (const file of files) {
const filepath = path.join(dir, file.name);
await this.processPath(file, filepath);
}
}
private async convertToDocuments(filepath: string) {
const content = await new MarkdownReader().loadData(filepath);
this.docs.push(...content);
}
async loadData(): Promise<Document[]> {
await this.readFromPath(this.inputDir);
return this.docs;
}
}
-30
View File
@@ -1,35 +1,5 @@
# @llamaindex/unit-test
## 0.0.34
### Patch Changes
- Updated dependencies [515f2c1]
- llamaindex@0.8.24
## 0.0.33
### Patch Changes
- @llamaindex/cloud@2.0.15
- @llamaindex/core@0.4.15
- llamaindex@0.8.23
- @llamaindex/node-parser@0.0.16
- @llamaindex/openai@0.1.40
- @llamaindex/readers@1.0.17
## 0.0.32
### Patch Changes
- Updated dependencies [819af45]
- llamaindex@0.8.22
- @llamaindex/cloud@2.0.14
- @llamaindex/core@0.4.14
- @llamaindex/node-parser@0.0.15
- @llamaindex/openai@0.1.39
- @llamaindex/readers@1.0.16
## 0.0.31
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/unit-test",
"private": true,
"version": "0.0.34",
"version": "0.0.31",
"type": "module",
"scripts": {
"test": "vitest run"