Compare commits

..

5 Commits

Author SHA1 Message Date
github-actions[bot] 9db36f7384 Release 0.8.21 (#1513)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-11-19 18:18:35 -08:00
fatmelon efa221116f feat: integrate with azure cosmos db mongo vCore (#1500)
Co-authored-by: root <root@CPC-yangq-FRSGK>
Co-authored-by: Alex Yang <himself65@outlook.com>
2024-11-19 17:58:25 -08:00
Alex Yang 83c3897539 fix: pinecone vector store (#1512) 2024-11-19 17:51:45 -08:00
github-actions[bot] e1cbce1f80 Release 0.8.20 (#1511)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-11-19 10:01:18 -08:00
Madarco 02b22da384 fix: supports vercel bundling (#1509)
Co-authored-by: Alex Yang <himself65@outlook.com>
2024-11-19 09:52:56 -08:00
35 changed files with 511 additions and 25 deletions
+15
View File
@@ -1,5 +1,20 @@
# docs
## 0.0.126
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.125
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.124
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "docs",
"version": "0.0.124",
"version": "0.0.126",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/doc
## 0.0.24
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.23
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.22
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.0.22",
"version": "0.0.24",
"private": true,
"scripts": {
"build": "pnpm run build:docs && next build",
+3
View File
@@ -1 +1,4 @@
POSTGRES_USER=runner
PINECONE_API_KEY=
PINECONE_INDEX_NAME=
PINECONE_NAMESPACE=
@@ -1,5 +1,20 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.117
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.116
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.115
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.115",
"version": "0.0.117",
"type": "module",
"private": true,
"scripts": {
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/next-agent-test
## 0.1.117
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.1.116
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.1.115
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.115",
"version": "0.1.117",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,20 @@
# test-edge-runtime
## 0.1.116
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.1.115
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.1.114
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.114",
"version": "0.1.116",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,20 @@
# @llamaindex/next-node-runtime
## 0.0.98
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.97
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.96
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.0.96",
"version": "0.0.98",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,20 @@
# @llamaindex/waku-query-engine-test
## 0.0.117
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.116
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.115
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.115",
"version": "0.0.117",
"type": "module",
"private": true,
"scripts": {
+66
View File
@@ -0,0 +1,66 @@
import { Document, MetadataMode } from "@llamaindex/core/schema";
import { config } from "dotenv";
import {
OpenAIEmbedding,
PineconeVectorStore,
VectorStoreIndex,
} from "llamaindex";
import assert from "node:assert";
import { test } from "node:test";
config({ path: [".env.local", ".env", ".env.ci"] });
await test("pinecone", async (t) => {
if (
!process.env.PINECONE_API_KEY ||
!process.env.PINECONE_NAMESPACE ||
!process.env.PINECONE_INDEX_NAME
) {
return t.skip(
"PINECONE_API_KEY, PINECONE_NAMESPACE, and PINECONE_INDEX_NAME must be set to run this test",
);
}
const openaiEmbedding = new OpenAIEmbedding({
model: "text-embedding-3-large",
});
const vectorStore = new PineconeVectorStore({
embeddingModel: openaiEmbedding,
});
t.after(async () => {
await vectorStore.clearIndex();
});
const index = await VectorStoreIndex.fromVectorStore(vectorStore);
const retriever = index.asRetriever({
similarityTopK: 3,
});
const text = "We are open from 9am to 5pm";
await vectorStore.add([
new Document({
text,
embedding: await openaiEmbedding.getTextEmbedding(text),
}),
]);
const results = await retriever.retrieve({
query: "When are you open?",
});
results.every((result) => {
assert.ok(result.node.embedding instanceof Array);
result.node.embedding.every((embedding, idx) =>
assert.ok(
typeof embedding === "number",
`Embedding at index ${idx} should be a number`,
),
);
assert.ok(typeof result.score === "number", "Score should be a number");
assert.ok(
result.node.getContent(MetadataMode.NONE).length > 0,
"Content should not be empty",
);
});
});
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/autotool
## 5.0.21
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 5.0.20
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 5.0.19
### Patch Changes
@@ -1,5 +1,22 @@
# @llamaindex/autotool-01-node-example
## 0.0.64
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
- @llamaindex/autotool@5.0.21
## 0.0.63
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
- @llamaindex/autotool@5.0.20
## 0.0.62
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.62"
"version": "0.0.64"
}
@@ -1,5 +1,22 @@
# @llamaindex/autotool-02-next-example
## 0.1.108
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
- @llamaindex/autotool@5.0.21
## 0.1.107
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
- @llamaindex/autotool@5.0.20
## 0.1.106
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool-02-next-example",
"private": true,
"version": "0.1.106",
"version": "0.1.108",
"scripts": {
"dev": "next dev",
"build": "next build",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool",
"type": "module",
"version": "5.0.19",
"version": "5.0.21",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/experimental
## 0.0.133
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.132
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.131
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.131",
"version": "0.0.133",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+13
View File
@@ -1,5 +1,18 @@
# llamaindex
## 0.8.21
### Patch Changes
- 83c3897: fix: pinecone vector store search
- efa2211: feat: add Azure Cosmos DB mongo vCore DocumentStore, IndexStore, KVStore
## 0.8.20
### Patch Changes
- 02b22da: fix: supports Vercel bundling
## 0.8.19
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.8.19",
"version": "0.8.21",
"license": "MIT",
"type": "module",
"keywords": [
@@ -0,0 +1,49 @@
import { MongoClient } from "mongodb";
import { AzureCosmosVCoreKVStore } from "../kvStore/AzureCosmosMongovCoreKVStore.js";
import { KVDocumentStore } from "./KVDocumentStore.js";
const DEFAULT_DATABASE = "DocumentStoreDB";
const DEFAULT_COLLECTION = "DocumentStoreCollection";
export interface AzureCosmosVCoreDocumentStoreArgs {
azureCosmosVCoreKVStore: AzureCosmosVCoreKVStore;
namespace?: string;
}
export class AzureCosmosVCoreDocumentStore extends KVDocumentStore {
constructor({
azureCosmosVCoreKVStore,
namespace,
}: AzureCosmosVCoreDocumentStoreArgs) {
super(azureCosmosVCoreKVStore, namespace);
}
/**
* Static method for creating an instance using a MongoClient.
* @returns Instance of AzureCosmosVCoreDocumentStore
* @param mongoClient - MongoClient instance
* @param dbName - Database name
* @param collectionName - Collection name
* @example
* ```ts
* const mongoClient = new MongoClient("mongodb://localhost:27017");
* const indexStore = AzureCosmosVCoreDocumentStore.fromMongoClient(mongoClient, "my_db", "my_collection");
* ```
*/
static fromMongoClient(
mongoClient: MongoClient,
dbName: string = DEFAULT_DATABASE,
collectionName: string = DEFAULT_COLLECTION,
) {
const azureCosmosVCoreKVStore = new AzureCosmosVCoreKVStore({
mongoClient,
dbName,
collectionName,
});
const namespace = `${dbName}.${collectionName}`;
return new AzureCosmosVCoreDocumentStore({
azureCosmosVCoreKVStore,
namespace,
});
}
}
+3
View File
@@ -3,12 +3,15 @@ export * from "@llamaindex/core/storage/doc-store";
export * from "@llamaindex/core/storage/index-store";
export * from "@llamaindex/core/storage/kv-store";
export * from "./chatStore/AzureCosmosNoSqlChatStore.js";
export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js";
export * from "./docStore/AzureCosmosNoSqlDocumentStore.js";
export { PostgresDocumentStore } from "./docStore/PostgresDocumentStore.js";
export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
export * from "./FileSystem.js";
export * from "./indexStore/AzureCosmosMongovCoreIndexStore.js";
export * from "./indexStore/AzureCosmosNoSqlIndexStore.js";
export { PostgresIndexStore } from "./indexStore/PostgresIndexStore.js";
export * from "./kvStore/AzureCosmosMongovCoreKVStore.js";
export * from "./kvStore/AzureCosmosNoSqlKVStore.js";
export { PostgresKVStore } from "./kvStore/PostgresKVStore.js";
export * from "./StorageContext.js";
@@ -0,0 +1,49 @@
import { MongoClient } from "mongodb";
import { AzureCosmosVCoreKVStore } from "../kvStore/AzureCosmosMongovCoreKVStore.js";
import { KVIndexStore } from "./KVIndexStore.js";
const DEFAULT_DATABASE = "IndexStoreDB";
const DEFAULT_COLLECTION = "IndexStoreCollection";
export interface AzureCosmosVCoreIndexStoreArgs {
azureCosmosVCoreKVStore: AzureCosmosVCoreKVStore;
namespace?: string;
}
export class AzureCosmosVCoreIndexStore extends KVIndexStore {
constructor({
azureCosmosVCoreKVStore,
namespace,
}: AzureCosmosVCoreIndexStoreArgs) {
super(azureCosmosVCoreKVStore, namespace);
}
/**
* Static method for creating an instance using a MongoClient.
* @returns Instance of AzureCosmosVCoreIndexStore
* @param mongoClient - MongoClient instance
* @param dbName - Database name
* @param collectionName - Collection name
* @example
* ```ts
* const mongoClient = new MongoClient("mongodb://localhost:27017");
* const indexStore = AzureCosmosVCoreIndexStore.fromMongoClient(mongoClient, "my_db", "my_collection");
* ```
*/
static fromMongoClient(
mongoClient: MongoClient,
dbName: string = DEFAULT_DATABASE,
collectionName: string = DEFAULT_COLLECTION,
) {
const azureCosmosVCoreKVStore = new AzureCosmosVCoreKVStore({
mongoClient,
dbName,
collectionName,
});
const namespace = `${dbName}.${collectionName}`;
return new AzureCosmosVCoreIndexStore({
azureCosmosVCoreKVStore,
namespace,
});
}
}
@@ -0,0 +1,88 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { BaseKVStore } from "@llamaindex/core/storage/kv-store";
import type { Collection } from "mongodb";
import { MongoClient } from "mongodb";
const DEFAULT_CHAT_DATABASE = "KVStoreDB";
const DEFAULT_CHAT_Collection = "KVStoreCollection";
export interface VcoreConnectionStringOptions
extends AzureCosmosVCoreKVStoreConfig {
connectionString?: string;
}
export interface AzureCosmosVCoreKVStoreConfig {
mongoClient?: MongoClient;
dbName?: string;
collectionName?: string;
}
export class AzureCosmosVCoreKVStore extends BaseKVStore {
private mongoClient: MongoClient;
private dbName: string;
private collectionName: string;
private collection?: Collection;
/**
* Create a new AzureCosmosDBNoSQLVectorStore instance.
*/
constructor({
mongoClient,
dbName = DEFAULT_CHAT_DATABASE,
collectionName = DEFAULT_CHAT_Collection,
}: AzureCosmosVCoreKVStoreConfig) {
super();
if (!mongoClient) {
throw new Error(
"MongoClient is required for AzureCosmosDBNoSQLVectorStore initialization",
);
}
this.mongoClient = mongoClient;
this.dbName = dbName;
this.collectionName = collectionName;
}
client(): MongoClient {
return this.mongoClient;
}
private async ensureCollection(): Promise<Collection> {
if (!this.collection) {
this.collection = this.mongoClient
.db(this.dbName)
.collection(this.collectionName);
}
return this.collection;
}
async put(key: string, val: Record<string, any>): Promise<void> {
const collection = await this.ensureCollection();
const insertResult = await collection.insertOne({
id: key,
messages: val,
});
}
async get(key: string): Promise<Record<string, any> | null> {
const collection = await this.ensureCollection();
const result = await collection.findOne({ id: key });
return result || null;
}
async getAll(): Promise<Record<string, Record<string, any>>> {
const collection = await this.ensureCollection();
const cursor = collection.find();
const output: Record<string, Record<string, any>> = {};
await cursor.forEach((item) => {
output[item.id] = item.messages;
});
return output;
}
async delete(key: string): Promise<boolean> {
const collection = await this.ensureCollection();
await collection.deleteOne({ id: key });
return true;
}
}
@@ -176,7 +176,12 @@ export class PGVectorStore extends BaseVectorStore {
if ("clientConfig" in config) {
this.clientConfig = config.clientConfig;
} else {
if (config.client.constructor.name.includes("Vercel")) {
if (
config.client.constructor.name.includes("Vercel") ||
(!!(config.client as VercelPool).connect &&
!!(config.client as VercelPool).query &&
!(config.client as Sql).unsafe)
) {
this.isDBConnected = true;
this.db = fromVercelPool(config.client as unknown as VercelPool);
} else if (typeof config.client === "function") {
@@ -26,6 +26,7 @@ type PineconeParams = {
chunkSize?: number;
namespace?: string;
textKey?: string;
apiKey?: string;
} & VectorStoreBaseParams;
/**
@@ -48,6 +49,8 @@ export class PineconeVectorStore extends BaseVectorStore {
chunkSize: number;
textKey: string;
apiKey: string;
constructor(params?: PineconeParams) {
super(params);
this.indexName =
@@ -57,12 +60,19 @@ export class PineconeVectorStore extends BaseVectorStore {
params?.chunkSize ??
Number.parseInt(getEnv("PINECONE_CHUNK_SIZE") ?? "100");
this.textKey = params?.textKey ?? "text";
const apiKey = params?.apiKey ?? getEnv("PINECONE_API_KEY");
if (!apiKey) {
throw new Error("PINECONE_API_KEY is required");
}
this.apiKey = apiKey;
}
private async getDb(): Promise<Pinecone> {
if (!this.db) {
const { Pinecone } = await import("@pinecone-database/pinecone");
this.db = await new Pinecone();
this.db = new Pinecone({
apiKey: this.apiKey,
});
}
return Promise.resolve(this.db);
@@ -100,7 +110,7 @@ export class PineconeVectorStore extends BaseVectorStore {
*/
async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
if (embeddingResults.length == 0) {
return Promise.resolve([]);
return [];
}
const idx: Index = await this.index();
@@ -110,10 +120,10 @@ export class PineconeVectorStore extends BaseVectorStore {
const chunk = nodes.slice(i, i + this.chunkSize);
const result = await this.saveChunk(idx, chunk);
if (!result) {
return Promise.reject(new Error("Failed to save chunk"));
throw new Error("Failed to save chunk");
}
}
return Promise.resolve([]);
return [];
}
protected async saveChunk(idx: Index, chunk: PineconeRecord[]) {
@@ -157,13 +167,19 @@ export class PineconeVectorStore extends BaseVectorStore {
topK: query.similarityTopK,
includeValues: true,
includeMetadata: true,
filter: filter,
};
if (filter) {
defaultOptions.filter = filter;
}
const idx = await this.index();
const results = await idx.query(defaultOptions);
const idList = results.matches.map((row) => row.id);
if (idList.length == 0) {
return { nodes: [], similarities: [], ids: [] };
}
const records: FetchResponse = await idx.fetch(idList);
const rows = Object.values(records.records);
@@ -179,13 +195,11 @@ export class PineconeVectorStore extends BaseVectorStore {
return node;
});
const ret = {
return {
nodes: nodes,
similarities: results.matches.map((row) => row.score || 999),
ids: results.matches.map((row) => row.id),
};
return Promise.resolve(ret);
}
/**
@@ -197,8 +211,8 @@ export class PineconeVectorStore extends BaseVectorStore {
return Promise.resolve();
}
toPineconeFilter(stdFilters?: MetadataFilters): object {
if (!stdFilters) return {};
toPineconeFilter(stdFilters?: MetadataFilters): object | undefined {
if (stdFilters == null) return undefined;
const transformCondition = (
condition: `${FilterCondition}` = "and",
+13 -1
View File
@@ -65,7 +65,19 @@ export function metadataDictToNode(
}
} else {
nodeObj = JSON.parse(nodeContent);
nodeObj.metadata = rest;
nodeObj = {
...rest,
...options?.fallback,
...nodeObj,
};
nodeObj.metadata = {
...(options?.fallback &&
"metadata" in options.fallback &&
typeof options.fallback.metadata === "object"
? options?.fallback.metadata
: {}),
...rest,
};
}
// Note: we're using the name of the class stored in `_node_type`
+15
View File
@@ -1,5 +1,20 @@
# @llamaindex/unit-test
## 0.0.31
### Patch Changes
- Updated dependencies [83c3897]
- Updated dependencies [efa2211]
- llamaindex@0.8.21
## 0.0.30
### Patch Changes
- Updated dependencies [02b22da]
- llamaindex@0.8.20
## 0.0.29
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/unit-test",
"private": true,
"version": "0.0.29",
"version": "0.0.31",
"type": "module",
"scripts": {
"test": "vitest run"