mirror of
https://github.com/run-llama/LlamaIndexTS.git
synced 2026-07-15 14:55:41 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Thuc Pham
|
954d1bd26b |
@@ -1,55 +1,5 @@
|
||||
# docs
|
||||
|
||||
## 0.0.92
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
- @llamaindex/examples@0.0.9
|
||||
|
||||
## 0.0.91
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.0.90
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.0.89
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.0.88
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.0.87
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.0.86
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "docs",
|
||||
"version": "0.0.92",
|
||||
"version": "0.0.86",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"docusaurus": "docusaurus",
|
||||
|
||||
@@ -1,16 +1,5 @@
|
||||
# examples
|
||||
|
||||
## 0.0.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
- @llamaindex/core@0.3.0
|
||||
|
||||
## 0.0.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {
|
||||
AstraDBVectorStore,
|
||||
Document,
|
||||
MetadataFilters,
|
||||
storageContextFromDefaults,
|
||||
VectorStoreIndex,
|
||||
} from "llamaindex";
|
||||
@@ -43,10 +42,8 @@ async function main() {
|
||||
const index = await VectorStoreIndex.fromDocuments(docs, {
|
||||
storageContext: ctx,
|
||||
});
|
||||
const preFilters: MetadataFilters = {
|
||||
filters: [{ key: "id", operator: "in", value: [123, 789] }],
|
||||
}; // try changing the filters to see the different results
|
||||
const queryEngine = index.asQueryEngine({ preFilters });
|
||||
|
||||
const queryEngine = index.asQueryEngine();
|
||||
const response = await queryEngine.query({
|
||||
query: "Describe AstraDB.",
|
||||
});
|
||||
|
||||
@@ -1,83 +1,57 @@
|
||||
import {
|
||||
ChromaVectorStore,
|
||||
Document,
|
||||
MetadataFilters,
|
||||
VectorStoreIndex,
|
||||
storageContextFromDefaults,
|
||||
} from "llamaindex";
|
||||
|
||||
const collectionName = "dogs_with_color";
|
||||
const collectionName = "dog_colors";
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const docs = [
|
||||
new Document({
|
||||
text: "The dog is brown",
|
||||
metadata: {
|
||||
dogId: "1",
|
||||
},
|
||||
}),
|
||||
new Document({
|
||||
text: "The dog is red",
|
||||
metadata: {
|
||||
dogId: "2",
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
console.log("Creating ChromaDB vector store");
|
||||
const chromaVS = new ChromaVectorStore({ collectionName });
|
||||
const index = await VectorStoreIndex.fromVectorStore(chromaVS);
|
||||
const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
|
||||
|
||||
const queryFn = async (filters?: MetadataFilters) => {
|
||||
console.log("\nQuerying dogs by filters: ", JSON.stringify(filters));
|
||||
const query = "List all colors of dogs";
|
||||
const queryEngine = index.asQueryEngine({
|
||||
preFilters: filters,
|
||||
similarityTopK: 3,
|
||||
});
|
||||
const response = await queryEngine.query({ query });
|
||||
console.log(response.toString());
|
||||
};
|
||||
console.log("Embedding documents and adding to index");
|
||||
const index = await VectorStoreIndex.fromDocuments(docs, {
|
||||
storageContext: ctx,
|
||||
});
|
||||
|
||||
await queryFn(); // red, brown, yellow
|
||||
await queryFn({ filters: [{ key: "dogId", value: "1", operator: "==" }] }); // brown
|
||||
await queryFn({ filters: [{ key: "dogId", value: "1", operator: "!=" }] }); // red, yellow
|
||||
await queryFn({
|
||||
filters: [
|
||||
{ key: "dogId", value: "1", operator: "==" },
|
||||
{ key: "dogId", value: "3", operator: "==" },
|
||||
],
|
||||
condition: "or",
|
||||
}); // brown, yellow
|
||||
await queryFn({
|
||||
filters: [{ key: "dogId", value: ["1", "2"], operator: "in" }],
|
||||
}); // red, brown
|
||||
console.log("Querying index");
|
||||
const queryEngine = index.asQueryEngine({
|
||||
preFilters: {
|
||||
filters: [
|
||||
{
|
||||
key: "dogId",
|
||||
value: "2",
|
||||
operator: "==",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const response = await queryEngine.query({
|
||||
query: "What is the color of the dog?",
|
||||
});
|
||||
console.log(response.toString());
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
async function generate() {
|
||||
const docs = [
|
||||
new Document({
|
||||
id_: "doc1",
|
||||
text: "The dog is brown",
|
||||
metadata: {
|
||||
dogId: "1",
|
||||
},
|
||||
}),
|
||||
new Document({
|
||||
id_: "doc2",
|
||||
text: "The dog is red",
|
||||
metadata: {
|
||||
dogId: "2",
|
||||
},
|
||||
}),
|
||||
new Document({
|
||||
id_: "doc3",
|
||||
text: "The dog is yellow",
|
||||
metadata: {
|
||||
dogId: "3",
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
console.log("Creating ChromaDB vector store");
|
||||
const chromaVS = new ChromaVectorStore({ collectionName });
|
||||
const ctx = await storageContextFromDefaults({ vectorStore: chromaVS });
|
||||
|
||||
console.log("Embedding documents and adding to index");
|
||||
await VectorStoreIndex.fromDocuments(docs, {
|
||||
storageContext: ctx,
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
await generate();
|
||||
await main();
|
||||
})();
|
||||
void main();
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@llamaindex/examples",
|
||||
"private": true,
|
||||
"version": "0.0.9",
|
||||
"version": "0.0.8",
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha256-js": "^5.2.0",
|
||||
"@azure/identity": "^4.4.1",
|
||||
"@datastax/astra-db-ts": "^1.4.1",
|
||||
"@llamaindex/core": "^0.3.0",
|
||||
"@llamaindex/core": "^0.2.0",
|
||||
"@notionhq/client": "^2.2.15",
|
||||
"@pinecone-database/pinecone": "^3.0.2",
|
||||
"@vercel/postgres": "^0.10.0",
|
||||
@@ -15,7 +15,7 @@
|
||||
"commander": "^12.1.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"js-tiktoken": "^1.0.14",
|
||||
"llamaindex": "^0.7.0",
|
||||
"llamaindex": "^0.6.0",
|
||||
"mongodb": "^6.7.0",
|
||||
"pathe": "^1.1.2",
|
||||
"postgres": "^3.4.4"
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import {
|
||||
BaseVectorStore,
|
||||
getResponseSynthesizer,
|
||||
OpenAI,
|
||||
OpenAIEmbedding,
|
||||
@@ -7,6 +6,7 @@ import {
|
||||
Settings,
|
||||
TextNode,
|
||||
VectorIndexRetriever,
|
||||
VectorStore,
|
||||
VectorStoreIndex,
|
||||
VectorStoreQuery,
|
||||
VectorStoreQueryResult,
|
||||
@@ -24,7 +24,7 @@ Settings.llm = new OpenAI({
|
||||
* Please do not use this class in production; it's only for demonstration purposes.
|
||||
*/
|
||||
class PineconeVectorStore<T extends RecordMetadata = RecordMetadata>
|
||||
implements BaseVectorStore
|
||||
implements VectorStore
|
||||
{
|
||||
storesText = true;
|
||||
isEmbeddingQuery = false;
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/autotool
|
||||
|
||||
## 4.0.0
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 3.0.22
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 3.0.21
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 3.0.20
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 3.0.19
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 3.0.18
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 3.0.17
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,60 +1,5 @@
|
||||
# @llamaindex/autotool-01-node-example
|
||||
|
||||
## 0.0.32
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
- @llamaindex/autotool@4.0.0
|
||||
|
||||
## 0.0.31
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
- @llamaindex/autotool@3.0.22
|
||||
|
||||
## 0.0.30
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
- @llamaindex/autotool@3.0.21
|
||||
|
||||
## 0.0.29
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
- @llamaindex/autotool@3.0.20
|
||||
|
||||
## 0.0.28
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
- @llamaindex/autotool@3.0.19
|
||||
|
||||
## 0.0.27
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
- @llamaindex/autotool@3.0.18
|
||||
|
||||
## 0.0.26
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -13,5 +13,5 @@
|
||||
"scripts": {
|
||||
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
|
||||
},
|
||||
"version": "0.0.32"
|
||||
"version": "0.0.26"
|
||||
}
|
||||
|
||||
@@ -1,60 +1,5 @@
|
||||
# @llamaindex/autotool-02-next-example
|
||||
|
||||
## 0.1.76
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
- @llamaindex/autotool@4.0.0
|
||||
|
||||
## 0.1.75
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
- @llamaindex/autotool@3.0.22
|
||||
|
||||
## 0.1.74
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
- @llamaindex/autotool@3.0.21
|
||||
|
||||
## 0.1.73
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
- @llamaindex/autotool@3.0.20
|
||||
|
||||
## 0.1.72
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
- @llamaindex/autotool@3.0.19
|
||||
|
||||
## 0.1.71
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
- @llamaindex/autotool@3.0.18
|
||||
|
||||
## 0.1.70
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/autotool-02-next-example",
|
||||
"private": true,
|
||||
"version": "0.1.76",
|
||||
"version": "0.1.70",
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/autotool",
|
||||
"type": "module",
|
||||
"version": "4.0.0",
|
||||
"version": "3.0.17",
|
||||
"description": "auto transpile your JS function to LLM Agent compatible",
|
||||
"files": [
|
||||
"dist",
|
||||
|
||||
@@ -1,20 +1,5 @@
|
||||
# @llamaindex/cloud
|
||||
|
||||
## 1.0.0
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- @llamaindex/core@0.3.0
|
||||
|
||||
## 0.2.14
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- @llamaindex/core@0.2.12
|
||||
|
||||
## 0.2.13
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/cloud",
|
||||
"version": "1.0.0",
|
||||
"version": "0.2.13",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,20 +1,5 @@
|
||||
# @llamaindex/community
|
||||
|
||||
## 0.0.48
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- @llamaindex/core@0.3.0
|
||||
|
||||
## 0.0.47
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- @llamaindex/core@0.2.12
|
||||
|
||||
## 0.0.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/community",
|
||||
"description": "Community package for LlamaIndexTS",
|
||||
"version": "0.0.48",
|
||||
"version": "0.0.46",
|
||||
"type": "module",
|
||||
"types": "dist/type/index.d.ts",
|
||||
"main": "dist/cjs/index.js",
|
||||
|
||||
@@ -1,18 +1,5 @@
|
||||
# @llamaindex/core
|
||||
|
||||
## 0.3.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 1364e8e: update metadata extractors to use PromptTemplate
|
||||
- 96fc69c: add defaultQuestionExtractPrompt
|
||||
|
||||
## 0.2.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 5f67820: Fix that node parsers generate nodes with UUIDs
|
||||
|
||||
## 0.2.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/core",
|
||||
"type": "module",
|
||||
"version": "0.3.0",
|
||||
"version": "0.2.11",
|
||||
"description": "LlamaIndex Core Module",
|
||||
"exports": {
|
||||
"./agent": {
|
||||
|
||||
@@ -12,15 +12,11 @@ export {
|
||||
defaultCondenseQuestionPrompt,
|
||||
defaultContextSystemPrompt,
|
||||
defaultKeywordExtractPrompt,
|
||||
defaultNodeTextTemplate,
|
||||
defaultQueryKeywordExtractPrompt,
|
||||
defaultQuestionExtractPrompt,
|
||||
defaultRefinePrompt,
|
||||
defaultSubQuestionPrompt,
|
||||
defaultSummaryPrompt,
|
||||
defaultTextQAPrompt,
|
||||
defaultTitleCombinePromptTemplate,
|
||||
defaultTitleExtractorPromptTemplate,
|
||||
defaultTreeSummarizePrompt,
|
||||
} from "./prompt";
|
||||
export type {
|
||||
@@ -29,12 +25,9 @@ export type {
|
||||
ContextSystemPrompt,
|
||||
KeywordExtractPrompt,
|
||||
QueryKeywordExtractPrompt,
|
||||
QuestionExtractPrompt,
|
||||
RefinePrompt,
|
||||
SubQuestionPrompt,
|
||||
SummaryPrompt,
|
||||
TextQAPrompt,
|
||||
TitleCombinePrompt,
|
||||
TitleExtractorPrompt,
|
||||
TreeSummarizePrompt,
|
||||
} from "./prompt";
|
||||
|
||||
@@ -13,12 +13,8 @@ export type CondenseQuestionPrompt = PromptTemplate<
|
||||
["chatHistory", "question"]
|
||||
>;
|
||||
export type ContextSystemPrompt = PromptTemplate<["context"]>;
|
||||
export type KeywordExtractPrompt = PromptTemplate<["context", "maxKeywords"]>;
|
||||
export type KeywordExtractPrompt = PromptTemplate<["context"]>;
|
||||
export type QueryKeywordExtractPrompt = PromptTemplate<["question"]>;
|
||||
export type QuestionExtractPrompt = PromptTemplate<["context", "numQuestions"]>;
|
||||
export type TitleExtractorPrompt = PromptTemplate<["context"]>;
|
||||
export type TitleCombinePrompt = PromptTemplate<["context"]>;
|
||||
export type KeywordExtractorPrompt = PromptTemplate<["context", "numKeywords"]>;
|
||||
|
||||
export const defaultTextQAPrompt: TextQAPrompt = new PromptTemplate({
|
||||
templateVars: ["context", "query"],
|
||||
@@ -257,55 +253,3 @@ export const defaultQueryKeywordExtractPrompt = new PromptTemplate({
|
||||
}).partialFormat({
|
||||
maxKeywords: "10",
|
||||
});
|
||||
|
||||
export const defaultQuestionExtractPrompt = new PromptTemplate({
|
||||
templateVars: ["numQuestions", "context"],
|
||||
template: `(
|
||||
"Given the contextual informations below, generate {numQuestions} questions this context can provides specific answers to which are unlikely to be found else where. Higher-level summaries of surrounding context may be provided as well. "
|
||||
"Try using these summaries to generate better questions that this context can answer."
|
||||
"---------------------"
|
||||
"{context}"
|
||||
"---------------------"
|
||||
"Provide questions in the following format: 'QUESTIONS: <questions>'"
|
||||
)`,
|
||||
}).partialFormat({
|
||||
numQuestions: "5",
|
||||
});
|
||||
|
||||
export const defaultTitleExtractorPromptTemplate = new PromptTemplate({
|
||||
templateVars: ["context"],
|
||||
template: `{context}
|
||||
Give a title that summarizes all of the unique entities, titles or themes found in the context.
|
||||
Title: `,
|
||||
});
|
||||
|
||||
export const defaultTitleCombinePromptTemplate = new PromptTemplate({
|
||||
templateVars: ["context"],
|
||||
template: `{context}
|
||||
Based on the above candidate titles and contents, what is the comprehensive title for this document?
|
||||
Title: `,
|
||||
});
|
||||
|
||||
export const defaultKeywordExtractorPromptTemplate = new PromptTemplate({
|
||||
templateVars: ["context", "numKeywords"],
|
||||
template: `{context}
|
||||
Give {numKeywords} unique keywords for this document.
|
||||
Format as comma separated.
|
||||
Keywords: `,
|
||||
}).partialFormat({
|
||||
keywordCount: "5",
|
||||
});
|
||||
|
||||
export const defaultNodeTextTemplate = new PromptTemplate({
|
||||
templateVars: ["metadataStr", "content"],
|
||||
template: `[Excerpt from document]
|
||||
{metadataStr}
|
||||
Excerpt:
|
||||
-----
|
||||
{content}
|
||||
-----
|
||||
`,
|
||||
}).partialFormat({
|
||||
metadataStr: "",
|
||||
content: "",
|
||||
});
|
||||
|
||||
@@ -479,7 +479,7 @@ export function buildNodeFromSplits(
|
||||
) {
|
||||
const imageDoc = doc as ImageNode;
|
||||
const imageNode = new ImageNode({
|
||||
id_: idGenerator(i, imageDoc),
|
||||
id_: imageDoc.id_ ?? idGenerator(i, imageDoc),
|
||||
text: textChunk,
|
||||
image: imageDoc.image,
|
||||
embedding: imageDoc.embedding,
|
||||
@@ -496,7 +496,7 @@ export function buildNodeFromSplits(
|
||||
) {
|
||||
const textDoc = doc as TextNode;
|
||||
const node = new TextNode({
|
||||
id_: idGenerator(i, textDoc),
|
||||
id_: textDoc.id_ ?? idGenerator(i, textDoc),
|
||||
text: textChunk,
|
||||
embedding: textDoc.embedding,
|
||||
excludedEmbedMetadataKeys: [...textDoc.excludedEmbedMetadataKeys],
|
||||
|
||||
@@ -80,3 +80,4 @@ export {
|
||||
} from "./llms";
|
||||
|
||||
export { objectEntries } from "./object-entries";
|
||||
export { UUIDFromString } from "./uuid";
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { createSHA256 } from "@llamaindex/env";
|
||||
|
||||
export function UUIDFromString(input: string) {
|
||||
const hashFunction = createSHA256();
|
||||
hashFunction.update(input);
|
||||
const base64Hash = hashFunction.digest();
|
||||
|
||||
// Convert base64 to hex
|
||||
const hexHash = Buffer.from(base64Hash, "base64").toString("hex");
|
||||
|
||||
// Format the hash to resemble a UUID (version 5 style)
|
||||
const uuid = [
|
||||
hexHash.substring(0, 8),
|
||||
hexHash.substring(8, 12),
|
||||
"5" + hexHash.substring(12, 15), // Set the version to 5 (name-based)
|
||||
((parseInt(hexHash.substring(15, 17), 16) & 0x3f) | 0x80).toString(16) +
|
||||
hexHash.substring(17, 19), // Set the variant
|
||||
hexHash.substring(19, 31),
|
||||
].join("-");
|
||||
|
||||
return uuid;
|
||||
}
|
||||
@@ -2,7 +2,6 @@ import {
|
||||
SentenceSplitter,
|
||||
splitBySentenceTokenizer,
|
||||
} from "@llamaindex/core/node-parser";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
describe("sentence splitter", () => {
|
||||
@@ -116,26 +115,4 @@ describe("sentence splitter", () => {
|
||||
const split = splitBySentenceTokenizer();
|
||||
expect(split(text)).toEqual([text]);
|
||||
});
|
||||
|
||||
test("split nodes with UUID IDs and correct relationships", () => {
|
||||
const UUID_REGEX =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const sentenceSplitter = new SentenceSplitter();
|
||||
const docId = "test-doc-id";
|
||||
const doc = new Document({
|
||||
id_: docId,
|
||||
text: "This is a test sentence. This is another test sentence.",
|
||||
});
|
||||
const nodes = sentenceSplitter.getNodesFromDocuments([doc]);
|
||||
nodes.forEach((node) => {
|
||||
// test node id should match uuid regex
|
||||
expect(node.id_).toMatch(UUID_REGEX);
|
||||
|
||||
// test source reference to the doc ID
|
||||
const source = node.relationships?.SOURCE;
|
||||
expect(source).toBeDefined();
|
||||
expect(source).toHaveProperty("nodeId");
|
||||
expect((source as { nodeId: string }).nodeId).toEqual(docId);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { UUIDFromString } from "@llamaindex/core/utils";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const UUID_REGEX =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
describe("UUIDFromString", () => {
|
||||
it("should convert string to UUID", () => {
|
||||
const string = "document_id_1";
|
||||
const result = UUIDFromString(string);
|
||||
expect(result).toBeDefined();
|
||||
expect(result).toMatch(UUID_REGEX);
|
||||
});
|
||||
|
||||
it("should return the same UUID for the same input string", () => {
|
||||
const string = "document_id_1";
|
||||
const result1 = UUIDFromString(string);
|
||||
const result2 = UUIDFromString(string);
|
||||
expect(result1).toEqual(result2);
|
||||
});
|
||||
|
||||
it("should return the different UUID for different input strings", () => {
|
||||
const string1 = "document_id_1";
|
||||
const string2 = "document_id_2";
|
||||
const result1 = UUIDFromString(string1);
|
||||
const result2 = UUIDFromString(string2);
|
||||
expect(result1).not.toEqual(result2);
|
||||
});
|
||||
|
||||
it("should handle case-sensitive input strings", () => {
|
||||
const string1 = "document_id_1";
|
||||
const string2 = "Document_Id_1";
|
||||
const result1 = UUIDFromString(string1);
|
||||
const result2 = UUIDFromString(string2);
|
||||
expect(result1).not.toEqual(result2);
|
||||
});
|
||||
});
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/experimental
|
||||
|
||||
## 0.0.101
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.0.100
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.0.99
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.0.98
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.0.97
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.0.96
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.0.95
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/experimental",
|
||||
"description": "Experimental package for LlamaIndexTS",
|
||||
"version": "0.0.101",
|
||||
"version": "0.0.95",
|
||||
"type": "module",
|
||||
"types": "dist/type/index.d.ts",
|
||||
"main": "dist/cjs/index.js",
|
||||
|
||||
@@ -1,67 +1,5 @@
|
||||
# llamaindex
|
||||
|
||||
## 0.7.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 1364e8e: update metadata extractors to use PromptTemplate
|
||||
- 96fc69c: Correct initialization of QuestionsAnsweredExtractor so that it uses the promptTemplate arg when passed in
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 3b7736f: feat: added gemini 002 support
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- @llamaindex/core@0.3.0
|
||||
- @llamaindex/cloud@1.0.0
|
||||
- @llamaindex/ollama@0.0.8
|
||||
- @llamaindex/openai@0.1.17
|
||||
- @llamaindex/groq@0.0.16
|
||||
|
||||
## 0.6.22
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 5729bd9: Fix LlamaCloud API calls for ensuring an index and for file uploads
|
||||
|
||||
## 0.6.21
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 6f75306: feat: support metadata filters for AstraDB
|
||||
- 94cb4ad: feat: Add metadata filters to ChromaDb and update to 1.9.2
|
||||
|
||||
## 0.6.20
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 6a9a7b1: fix: take init api key into account
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- @llamaindex/openai@0.1.16
|
||||
- @llamaindex/groq@0.0.15
|
||||
|
||||
## 0.6.19
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 62cba52: Add ensureIndex function to LlamaCloudIndex
|
||||
- d265e96: fix: ignore resolving unpdf for nextjs
|
||||
- d30bbf7: Convert undefined values to null in LlamaCloud filters
|
||||
- 53fd00a: Fix getPipelineId in LlamaCloudIndex
|
||||
|
||||
## 0.6.18
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 5f67820: Fix that node parsers generate nodes with UUIDs
|
||||
- fe08d04: Fix LlamaCloud retrieval with multiple pipelines
|
||||
- Updated dependencies [5f67820]
|
||||
- @llamaindex/core@0.2.12
|
||||
- @llamaindex/cloud@0.2.14
|
||||
- @llamaindex/ollama@0.0.7
|
||||
- @llamaindex/openai@0.1.15
|
||||
- @llamaindex/groq@0.0.14
|
||||
|
||||
## 0.6.17
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/cloudflare-worker-agent-test
|
||||
|
||||
## 0.0.85
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.0.84
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.0.83
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.0.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.0.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.0.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.0.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/cloudflare-worker-agent-test",
|
||||
"version": "0.0.85",
|
||||
"version": "0.0.79",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
# @llamaindex/llama-parse-browser-test
|
||||
|
||||
## 0.0.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @llamaindex/cloud@1.0.0
|
||||
|
||||
## 0.0.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @llamaindex/cloud@0.2.14
|
||||
|
||||
## 0.0.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/llama-parse-browser-test",
|
||||
"private": true,
|
||||
"version": "0.0.11",
|
||||
"version": "0.0.9",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/next-agent-test
|
||||
|
||||
## 0.1.85
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.1.84
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.1.83
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.1.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.1.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.1.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.1.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/next-agent-test",
|
||||
"version": "0.1.85",
|
||||
"version": "0.1.79",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# test-edge-runtime
|
||||
|
||||
## 0.1.84
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.1.83
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.1.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.1.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.1.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.1.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.1.78
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/nextjs-edge-runtime-test",
|
||||
"version": "0.1.84",
|
||||
"version": "0.1.78",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/next-node-runtime
|
||||
|
||||
## 0.0.66
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.0.65
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.0.64
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.0.63
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.0.62
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.0.61
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.0.60
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/next-node-runtime-test",
|
||||
"version": "0.0.66",
|
||||
"version": "0.0.60",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,54 +1,5 @@
|
||||
# @llamaindex/waku-query-engine-test
|
||||
|
||||
## 0.0.85
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [3b7736f]
|
||||
- Updated dependencies [96fc69c]
|
||||
- llamaindex@0.7.0
|
||||
|
||||
## 0.0.84
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5729bd9]
|
||||
- llamaindex@0.6.22
|
||||
|
||||
## 0.0.83
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6f75306]
|
||||
- Updated dependencies [94cb4ad]
|
||||
- llamaindex@0.6.21
|
||||
|
||||
## 0.0.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- llamaindex@0.6.20
|
||||
|
||||
## 0.0.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [62cba52]
|
||||
- Updated dependencies [d265e96]
|
||||
- Updated dependencies [d30bbf7]
|
||||
- Updated dependencies [53fd00a]
|
||||
- llamaindex@0.6.19
|
||||
|
||||
## 0.0.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- Updated dependencies [fe08d04]
|
||||
- llamaindex@0.6.18
|
||||
|
||||
## 0.0.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/waku-query-engine-test",
|
||||
"version": "0.0.85",
|
||||
"version": "0.0.79",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -56,11 +56,26 @@ await test("init without client", async (t) => {
|
||||
assert.notDeepStrictEqual(db, undefined);
|
||||
});
|
||||
|
||||
await test("should able to insert nodes with ID is not UUID", async (t) => {
|
||||
const node = new Document({
|
||||
id_: "./data/planets.md_1",
|
||||
text: "hello world",
|
||||
embedding: [0.1, 0.2, 0.3],
|
||||
});
|
||||
const vectorStore = new PGVectorStore({ clientConfig: pgConfig });
|
||||
const db = await vectorStore.client();
|
||||
t.after(async () => {
|
||||
await db.close();
|
||||
});
|
||||
const result = await vectorStore.add([node]);
|
||||
assert.ok(result.length > 0, "Result should have at least one item");
|
||||
});
|
||||
|
||||
await test("simple node", async (t) => {
|
||||
const dimensions = 3;
|
||||
const schemaName =
|
||||
"llamaindex_vector_store_test_" + Math.random().toString(36).substring(7);
|
||||
const nodeId = "5bb16627-f6c0-459c-bb18-71642813ef21";
|
||||
const nodeId = "./data/planets.md_1";
|
||||
const node = new Document({
|
||||
text: "hello world",
|
||||
id_: nodeId,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llamaindex",
|
||||
"version": "0.7.0",
|
||||
"version": "0.6.17",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"keywords": [
|
||||
@@ -48,7 +48,7 @@
|
||||
"@zilliz/milvus2-sdk-node": "^2.4.6",
|
||||
"ajv": "^8.17.1",
|
||||
"assemblyai": "^4.7.0",
|
||||
"chromadb": "1.9.2",
|
||||
"chromadb": "1.8.1",
|
||||
"cohere-ai": "7.13.0",
|
||||
"discord-api-types": "^0.37.98",
|
||||
"groq-sdk": "^0.6.1",
|
||||
|
||||
@@ -41,7 +41,7 @@ export class LLamaCloudFileService {
|
||||
) {
|
||||
initService();
|
||||
const { data: file } = await FilesService.uploadFileApiV1FilesPost({
|
||||
query: { project_id: projectId },
|
||||
path: { project_id: projectId },
|
||||
body: {
|
||||
upload_file: uploadFile,
|
||||
},
|
||||
@@ -85,7 +85,7 @@ export class LLamaCloudFileService {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100)); // Sleep for 100ms
|
||||
}
|
||||
throw new Error(
|
||||
`File processing did not complete after ${maxAttempts} attempts. Check your LlamaCloud index at https://cloud.llamaindex.ai/project/${projectId}/deploy/${pipelineId} for more details.`,
|
||||
`File processing did not complete after ${maxAttempts} attempts.`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
|
||||
import type { BaseSynthesizer } from "@llamaindex/core/response-synthesizers";
|
||||
import type { Document } from "@llamaindex/core/schema";
|
||||
import type { Document, TransformComponent } from "@llamaindex/core/schema";
|
||||
import { RetrieverQueryEngine } from "../engines/query/RetrieverQueryEngine.js";
|
||||
import type { BaseNodePostprocessor } from "../postprocessors/types.js";
|
||||
import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
|
||||
import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
|
||||
import { getPipelineCreate } from "./config.js";
|
||||
import type { CloudConstructorParams } from "./type.js";
|
||||
import {
|
||||
getAppBaseUrl,
|
||||
getPipelineId,
|
||||
getProjectId,
|
||||
initService,
|
||||
} from "./utils.js";
|
||||
import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
|
||||
|
||||
import { PipelinesService, type PipelineCreate } from "@llamaindex/cloud/api";
|
||||
import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
|
||||
import { SentenceSplitter } from "@llamaindex/core/node-parser";
|
||||
import type { BaseRetriever } from "@llamaindex/core/retriever";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import { OpenAIEmbedding } from "@llamaindex/openai";
|
||||
import { Settings } from "../Settings.js";
|
||||
|
||||
export class LlamaCloudIndex {
|
||||
@@ -30,7 +28,10 @@ export class LlamaCloudIndex {
|
||||
verbose = Settings.debug,
|
||||
raiseOnError = false,
|
||||
): Promise<void> {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
const pipelineId = await this.getPipelineId(
|
||||
this.params.name,
|
||||
this.params.projectName,
|
||||
);
|
||||
|
||||
if (verbose) {
|
||||
console.log("Waiting for pipeline ingestion: ");
|
||||
@@ -77,7 +78,10 @@ export class LlamaCloudIndex {
|
||||
verbose = Settings.debug,
|
||||
raiseOnError = false,
|
||||
): Promise<void> {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
const pipelineId = await this.getPipelineId(
|
||||
this.params.name,
|
||||
this.params.projectName,
|
||||
);
|
||||
|
||||
if (verbose) {
|
||||
console.log("Loading data: ");
|
||||
@@ -139,13 +143,17 @@ export class LlamaCloudIndex {
|
||||
public async getPipelineId(
|
||||
name?: string,
|
||||
projectName?: string,
|
||||
organizationId?: string,
|
||||
): Promise<string> {
|
||||
return await getPipelineId(
|
||||
name ?? this.params.name,
|
||||
projectName ?? this.params.projectName,
|
||||
organizationId ?? this.params.organizationId,
|
||||
);
|
||||
const { data: pipelines } =
|
||||
await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
path: {
|
||||
project_id: await this.getProjectId(projectName),
|
||||
project_name: name ?? this.params.name,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
return pipelines[0]!.id;
|
||||
}
|
||||
|
||||
public async getProjectId(
|
||||
@@ -158,42 +166,75 @@ export class LlamaCloudIndex {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds documents to the given index parameters. If the index does not exist, it will be created.
|
||||
*
|
||||
* @param params - An object containing the following properties:
|
||||
* - documents: An array of Document objects to be added to the index.
|
||||
* - verbose: Optional boolean to enable verbose logging.
|
||||
* - Additional properties from CloudConstructorParams.
|
||||
* @returns A Promise that resolves to a new LlamaCloudIndex instance.
|
||||
*/
|
||||
static async fromDocuments(
|
||||
params: {
|
||||
documents: Document[];
|
||||
transformations?: TransformComponent[];
|
||||
verbose?: boolean;
|
||||
} & CloudConstructorParams,
|
||||
config?: {
|
||||
embedding: PipelineCreate["embedding_config"];
|
||||
transform: PipelineCreate["transform_config"];
|
||||
},
|
||||
): Promise<LlamaCloudIndex> {
|
||||
const index = new LlamaCloudIndex({ ...params });
|
||||
await index.ensureIndex({ ...config, verbose: params.verbose ?? false });
|
||||
await index.addDocuments(params.documents, params.verbose);
|
||||
return index;
|
||||
}
|
||||
|
||||
async addDocuments(documents: Document[], verbose?: boolean): Promise<void> {
|
||||
initService(params);
|
||||
const defaultTransformations: TransformComponent[] = [
|
||||
new SentenceSplitter(),
|
||||
new OpenAIEmbedding({
|
||||
apiKey: getEnv("OPENAI_API_KEY"),
|
||||
}),
|
||||
];
|
||||
const apiUrl = getAppBaseUrl();
|
||||
const projectId = await this.getProjectId();
|
||||
const pipelineId = await this.getPipelineId();
|
||||
|
||||
const pipelineCreateParams = await getPipelineCreate({
|
||||
pipelineName: params.name,
|
||||
pipelineType: "MANAGED",
|
||||
inputNodes: params.documents,
|
||||
transformations: params.transformations ?? defaultTransformations,
|
||||
});
|
||||
|
||||
const { data: project } =
|
||||
await ProjectsService.upsertProjectApiV1ProjectsPut({
|
||||
path: {
|
||||
organization_id: params.organizationId,
|
||||
},
|
||||
body: {
|
||||
name: params.projectName ?? "default",
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (!project.id) {
|
||||
throw new Error("Project ID should be defined");
|
||||
}
|
||||
|
||||
const { data: pipeline } =
|
||||
await PipelinesService.upsertPipelineApiV1PipelinesPut({
|
||||
path: {
|
||||
project_id: project.id,
|
||||
},
|
||||
body: pipelineCreateParams.configured_transformations
|
||||
? {
|
||||
name: params.name,
|
||||
configured_transformations:
|
||||
pipelineCreateParams.configured_transformations,
|
||||
}
|
||||
: {
|
||||
name: params.name,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (!pipeline.id) {
|
||||
throw new Error("Pipeline ID must be defined");
|
||||
}
|
||||
|
||||
if (params.verbose) {
|
||||
console.log(`Created pipeline ${pipeline.id} with name ${params.name}`);
|
||||
}
|
||||
|
||||
await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
|
||||
{
|
||||
path: {
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_id: pipeline.id,
|
||||
},
|
||||
body: documents.map((doc) => ({
|
||||
body: params.documents.map((doc) => ({
|
||||
metadata: doc.metadata,
|
||||
text: doc.text,
|
||||
excluded_embed_metadata_keys: doc.excludedEmbedMetadataKeys,
|
||||
@@ -207,7 +248,7 @@ export class LlamaCloudIndex {
|
||||
const { data: pipelineStatus } =
|
||||
await PipelinesService.getPipelineStatusApiV1PipelinesPipelineIdStatusGet(
|
||||
{
|
||||
path: { pipeline_id: pipelineId },
|
||||
path: { pipeline_id: pipeline.id },
|
||||
throwOnError: true,
|
||||
},
|
||||
);
|
||||
@@ -221,30 +262,32 @@ export class LlamaCloudIndex {
|
||||
|
||||
if (pipelineStatus.status === "ERROR") {
|
||||
console.error(
|
||||
`Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
|
||||
`Some documents failed to ingest, check your pipeline logs at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
|
||||
);
|
||||
throw new Error("Some documents failed to ingest");
|
||||
}
|
||||
|
||||
if (pipelineStatus.status === "PARTIAL_SUCCESS") {
|
||||
console.info(
|
||||
`Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
|
||||
`Documents ingestion partially succeeded, to check a more complete status check your pipeline at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
if (params.verbose) {
|
||||
process.stdout.write(".");
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
if (params.verbose) {
|
||||
console.info(
|
||||
`Ingestion completed, find your index at ${apiUrl}/project/${projectId}/deploy/${pipelineId}`,
|
||||
`Ingestion completed, find your index at ${apiUrl}/project/${project.id}/deploy/${pipeline.id}`,
|
||||
);
|
||||
}
|
||||
|
||||
return new LlamaCloudIndex({ ...params });
|
||||
}
|
||||
|
||||
asRetriever(params: CloudRetrieveParams = {}): BaseRetriever {
|
||||
@@ -270,7 +313,14 @@ export class LlamaCloudIndex {
|
||||
}
|
||||
|
||||
async insert(document: Document) {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
const pipelineId = await this.getPipelineId(
|
||||
this.params.name,
|
||||
this.params.projectName,
|
||||
);
|
||||
|
||||
if (!pipelineId) {
|
||||
throw new Error("We couldn't find the pipeline ID for the given name");
|
||||
}
|
||||
|
||||
await PipelinesService.createBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPost(
|
||||
{
|
||||
@@ -293,7 +343,14 @@ export class LlamaCloudIndex {
|
||||
}
|
||||
|
||||
async delete(document: Document) {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
const pipelineId = await this.getPipelineId(
|
||||
this.params.name,
|
||||
this.params.projectName,
|
||||
);
|
||||
|
||||
if (!pipelineId) {
|
||||
throw new Error("We couldn't find the pipeline ID for the given name");
|
||||
}
|
||||
|
||||
await PipelinesService.deletePipelineDocumentApiV1PipelinesPipelineIdDocumentsDocumentIdDelete(
|
||||
{
|
||||
@@ -308,7 +365,14 @@ export class LlamaCloudIndex {
|
||||
}
|
||||
|
||||
async refreshDoc(document: Document) {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
const pipelineId = await this.getPipelineId(
|
||||
this.params.name,
|
||||
this.params.projectName,
|
||||
);
|
||||
|
||||
if (!pipelineId) {
|
||||
throw new Error("We couldn't find the pipeline ID for the given name");
|
||||
}
|
||||
|
||||
await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
|
||||
{
|
||||
@@ -329,71 +393,4 @@ export class LlamaCloudIndex {
|
||||
|
||||
await this.waitForDocumentIngestion([document.id_]);
|
||||
}
|
||||
|
||||
public async ensureIndex(config?: {
|
||||
embedding?: PipelineCreate["embedding_config"];
|
||||
transform?: PipelineCreate["transform_config"];
|
||||
verbose?: boolean;
|
||||
}): Promise<void> {
|
||||
const projectId = await this.getProjectId();
|
||||
|
||||
const { data: pipelines } =
|
||||
await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
query: {
|
||||
project_id: projectId,
|
||||
pipeline_name: this.params.name,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (pipelines.length === 0) {
|
||||
// no pipeline found, create a new one
|
||||
let embeddingConfig = config?.embedding;
|
||||
if (!embeddingConfig) {
|
||||
// no embedding config provided, use OpenAI as default
|
||||
const openAIApiKey = getEnv("OPENAI_API_KEY");
|
||||
const embeddingModel = getEnv("EMBEDDING_MODEL");
|
||||
if (!openAIApiKey || !embeddingModel) {
|
||||
throw new Error(
|
||||
"No embedding configuration provided. Fallback to OpenAI embedding model. OPENAI_API_KEY and EMBEDDING_MODEL environment variables must be set.",
|
||||
);
|
||||
}
|
||||
embeddingConfig = {
|
||||
type: "OPENAI_EMBEDDING",
|
||||
component: {
|
||||
api_key: openAIApiKey,
|
||||
model_name: embeddingModel,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let transformConfig = config?.transform;
|
||||
if (!transformConfig) {
|
||||
transformConfig = {
|
||||
mode: "auto",
|
||||
chunk_size: 1024,
|
||||
chunk_overlap: 200,
|
||||
};
|
||||
}
|
||||
|
||||
const { data: pipeline } =
|
||||
await PipelinesService.upsertPipelineApiV1PipelinesPut({
|
||||
query: {
|
||||
project_id: projectId,
|
||||
},
|
||||
body: {
|
||||
name: this.params.name,
|
||||
embedding_config: embeddingConfig,
|
||||
transform_config: transformConfig,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (config?.verbose) {
|
||||
console.log(
|
||||
`Created pipeline ${pipeline.id} with name ${pipeline.name}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import {
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
PipelinesService,
|
||||
type RetrievalParams,
|
||||
@@ -12,7 +11,7 @@ import type { NodeWithScore } from "@llamaindex/core/schema";
|
||||
import { jsonToNode, ObjectType } from "@llamaindex/core/schema";
|
||||
import { extractText } from "@llamaindex/core/utils";
|
||||
import type { ClientParams, CloudConstructorParams } from "./type.js";
|
||||
import { getPipelineId, initService } from "./utils.js";
|
||||
import { getProjectId, initService } from "./utils.js";
|
||||
|
||||
export type CloudRetrieveParams = Omit<
|
||||
RetrievalParams,
|
||||
@@ -43,24 +42,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
});
|
||||
}
|
||||
|
||||
// LlamaCloud expects null values for filters, but LlamaIndexTS uses undefined for empty values
|
||||
// This function converts the undefined values to null
|
||||
private convertFilter(filters?: MetadataFilters): MetadataFilters | null {
|
||||
if (!filters) return null;
|
||||
|
||||
const processFilter = (
|
||||
filter: MetadataFilter | MetadataFilters,
|
||||
): MetadataFilter | MetadataFilters => {
|
||||
if ("filters" in filter) {
|
||||
// type MetadataFilters
|
||||
return { ...filter, filters: filter.filters.map(processFilter) };
|
||||
}
|
||||
return { ...filter, value: filter.value ?? null };
|
||||
};
|
||||
|
||||
return { ...filters, filters: filters.filters.map(processFilter) };
|
||||
}
|
||||
|
||||
constructor(params: CloudConstructorParams & CloudRetrieveParams) {
|
||||
super();
|
||||
this.clientParams = { apiKey: params.apiKey, baseUrl: params.baseUrl };
|
||||
@@ -76,24 +57,45 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
}
|
||||
|
||||
async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
|
||||
const pipelineId = await getPipelineId(
|
||||
this.pipelineName,
|
||||
this.projectName,
|
||||
this.organizationId,
|
||||
);
|
||||
const { data: pipelines } =
|
||||
await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
query: {
|
||||
project_id: await getProjectId(this.projectName, this.organizationId),
|
||||
project_name: this.pipelineName,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
const filters = this.convertFilter(this.retrieveParams.filters);
|
||||
if (pipelines.length === 0 || !pipelines[0]!.id) {
|
||||
throw new Error(
|
||||
`No pipeline found with name ${this.pipelineName} in project ${this.projectName}`,
|
||||
);
|
||||
}
|
||||
|
||||
const { data: pipeline } =
|
||||
await PipelinesService.getPipelineApiV1PipelinesPipelineIdGet({
|
||||
path: {
|
||||
pipeline_id: pipelines[0]!.id,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (!pipeline) {
|
||||
throw new Error(
|
||||
`No pipeline found with name ${this.pipelineName} in project ${this.projectName}`,
|
||||
);
|
||||
}
|
||||
|
||||
const { data: results } =
|
||||
await PipelinesService.runSearchApiV1PipelinesPipelineIdRetrievePost({
|
||||
throwOnError: true,
|
||||
path: {
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_id: pipeline.id,
|
||||
},
|
||||
body: {
|
||||
...this.retrieveParams,
|
||||
query: extractText(query),
|
||||
search_filters: filters,
|
||||
search_filters: this.retrieveParams.filters as MetadataFilters,
|
||||
dense_similarity_top_k: this.retrieveParams.similarityTopK!,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import type {
|
||||
ConfiguredTransformationItem,
|
||||
PipelineCreate,
|
||||
PipelineType,
|
||||
} from "@llamaindex/cloud/api";
|
||||
import { SentenceSplitter } from "@llamaindex/core/node-parser";
|
||||
import { BaseNode, type TransformComponent } from "@llamaindex/core/schema";
|
||||
import { OpenAIEmbedding } from "@llamaindex/openai";
|
||||
|
||||
export type GetPipelineCreateParams = {
|
||||
pipelineName: string;
|
||||
pipelineType: PipelineType;
|
||||
transformations?: TransformComponent[];
|
||||
inputNodes?: BaseNode[];
|
||||
};
|
||||
|
||||
function getTransformationConfig(
|
||||
transformation: TransformComponent,
|
||||
): ConfiguredTransformationItem {
|
||||
if (transformation instanceof SentenceSplitter) {
|
||||
return {
|
||||
configurable_transformation_type: "SENTENCE_AWARE_NODE_PARSER",
|
||||
component: {
|
||||
chunk_size: transformation.chunkSize, // TODO: set to public in SentenceSplitter
|
||||
chunk_overlap: transformation.chunkOverlap, // TODO: set to public in SentenceSplitter
|
||||
include_metadata: transformation.includeMetadata,
|
||||
include_prev_next_rel: transformation.includePrevNextRel,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (transformation instanceof OpenAIEmbedding) {
|
||||
return {
|
||||
configurable_transformation_type: "OPENAI_EMBEDDING",
|
||||
component: {
|
||||
model: transformation.model,
|
||||
api_key: transformation.apiKey,
|
||||
embed_batch_size: transformation.embedBatchSize,
|
||||
dimensions: transformation.dimensions,
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`Unsupported transformation: ${typeof transformation}`);
|
||||
}
|
||||
|
||||
export async function getPipelineCreate(
|
||||
params: GetPipelineCreateParams,
|
||||
): Promise<PipelineCreate> {
|
||||
const { pipelineName, pipelineType, transformations = [] } = params;
|
||||
|
||||
return {
|
||||
name: pipelineName,
|
||||
configured_transformations: transformations.map(getTransformationConfig),
|
||||
pipeline_type: pipelineType,
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,4 @@
|
||||
import {
|
||||
client,
|
||||
PipelinesService,
|
||||
ProjectsService,
|
||||
} from "@llamaindex/cloud/api";
|
||||
import { client, ProjectsService } from "@llamaindex/cloud/api";
|
||||
import { DEFAULT_BASE_URL } from "@llamaindex/core/global";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ClientParams } from "./type.js";
|
||||
@@ -44,9 +40,9 @@ export async function getProjectId(
|
||||
): Promise<string> {
|
||||
const { data: projects } = await ProjectsService.listProjectsApiV1ProjectsGet(
|
||||
{
|
||||
query: {
|
||||
path: {
|
||||
project_name: projectName,
|
||||
organization_id: organizationId ?? null,
|
||||
organization_id: organizationId,
|
||||
},
|
||||
throwOnError: true,
|
||||
},
|
||||
@@ -70,26 +66,3 @@ export async function getProjectId(
|
||||
|
||||
return project.id;
|
||||
}
|
||||
|
||||
export async function getPipelineId(
|
||||
name: string,
|
||||
projectName: string,
|
||||
organizationId?: string,
|
||||
): Promise<string> {
|
||||
const { data: pipelines } =
|
||||
await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
query: {
|
||||
project_id: await getProjectId(projectName, organizationId),
|
||||
pipeline_name: name,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (pipelines.length === 0 || !pipelines[0]!.id) {
|
||||
throw new Error(
|
||||
`No pipeline found with name ${name} in project ${projectName}`,
|
||||
);
|
||||
}
|
||||
|
||||
return pipelines[0]!.id;
|
||||
}
|
||||
|
||||
@@ -1,20 +1,14 @@
|
||||
import type { LLM } from "@llamaindex/core/llms";
|
||||
import {
|
||||
PromptTemplate,
|
||||
defaultKeywordExtractPrompt,
|
||||
defaultQuestionExtractPrompt,
|
||||
defaultSummaryPrompt,
|
||||
defaultTitleCombinePromptTemplate,
|
||||
defaultTitleExtractorPromptTemplate,
|
||||
type KeywordExtractPrompt,
|
||||
type QuestionExtractPrompt,
|
||||
type SummaryPrompt,
|
||||
type TitleCombinePrompt,
|
||||
type TitleExtractorPrompt,
|
||||
} from "@llamaindex/core/prompts";
|
||||
import type { BaseNode } from "@llamaindex/core/schema";
|
||||
import { MetadataMode, TextNode } from "@llamaindex/core/schema";
|
||||
import { OpenAI } from "@llamaindex/openai";
|
||||
import {
|
||||
defaultKeywordExtractorPromptTemplate,
|
||||
defaultQuestionAnswerPromptTemplate,
|
||||
defaultSummaryExtractorPromptTemplate,
|
||||
defaultTitleCombinePromptTemplate,
|
||||
defaultTitleExtractorPromptTemplate,
|
||||
} from "./prompts.js";
|
||||
import { BaseExtractor } from "./types.js";
|
||||
|
||||
const STRIP_REGEX = /(\r\n|\n|\r)/gm;
|
||||
@@ -22,7 +16,6 @@ const STRIP_REGEX = /(\r\n|\n|\r)/gm;
|
||||
type KeywordExtractArgs = {
|
||||
llm?: LLM;
|
||||
keywords?: number;
|
||||
promptTemplate?: KeywordExtractPrompt["template"];
|
||||
};
|
||||
|
||||
type ExtractKeyword = {
|
||||
@@ -46,12 +39,6 @@ export class KeywordExtractor extends BaseExtractor {
|
||||
*/
|
||||
keywords: number = 5;
|
||||
|
||||
/**
|
||||
* The prompt template to use for the question extractor.
|
||||
* @type {string}
|
||||
*/
|
||||
promptTemplate: KeywordExtractPrompt;
|
||||
|
||||
/**
|
||||
* Constructor for the KeywordExtractor class.
|
||||
* @param {LLM} llm LLM instance.
|
||||
@@ -66,12 +53,6 @@ export class KeywordExtractor extends BaseExtractor {
|
||||
|
||||
this.llm = options?.llm ?? new OpenAI();
|
||||
this.keywords = options?.keywords ?? 5;
|
||||
this.promptTemplate = options?.promptTemplate
|
||||
? new PromptTemplate({
|
||||
templateVars: ["context", "maxKeywords"],
|
||||
template: options.promptTemplate,
|
||||
})
|
||||
: defaultKeywordExtractPrompt;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -85,9 +66,9 @@ export class KeywordExtractor extends BaseExtractor {
|
||||
}
|
||||
|
||||
const completion = await this.llm.complete({
|
||||
prompt: this.promptTemplate.format({
|
||||
context: node.getContent(MetadataMode.ALL),
|
||||
maxKeywords: this.keywords.toString(),
|
||||
prompt: defaultKeywordExtractorPromptTemplate({
|
||||
contextStr: node.getContent(MetadataMode.ALL),
|
||||
keywords: this.keywords,
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -112,8 +93,8 @@ export class KeywordExtractor extends BaseExtractor {
|
||||
type TitleExtractorsArgs = {
|
||||
llm?: LLM;
|
||||
nodes?: number;
|
||||
nodeTemplate?: TitleExtractorPrompt["template"];
|
||||
combineTemplate?: TitleCombinePrompt["template"];
|
||||
nodeTemplate?: string;
|
||||
combineTemplate?: string;
|
||||
};
|
||||
|
||||
type ExtractTitle = {
|
||||
@@ -148,19 +129,19 @@ export class TitleExtractor extends BaseExtractor {
|
||||
* The prompt template to use for the title extractor.
|
||||
* @type {string}
|
||||
*/
|
||||
nodeTemplate: TitleExtractorPrompt;
|
||||
nodeTemplate: string;
|
||||
|
||||
/**
|
||||
* The prompt template to merge title with..
|
||||
* @type {string}
|
||||
*/
|
||||
combineTemplate: TitleCombinePrompt;
|
||||
combineTemplate: string;
|
||||
|
||||
/**
|
||||
* Constructor for the TitleExtractor class.
|
||||
* @param {LLM} llm LLM instance.
|
||||
* @param {number} nodes Number of nodes to extract titles from.
|
||||
* @param {TitleExtractorPrompt} nodeTemplate The prompt template to use for the title extractor.
|
||||
* @param {string} nodeTemplate The prompt template to use for the title extractor.
|
||||
* @param {string} combineTemplate The prompt template to merge title with..
|
||||
*/
|
||||
constructor(options?: TitleExtractorsArgs) {
|
||||
@@ -169,19 +150,10 @@ export class TitleExtractor extends BaseExtractor {
|
||||
this.llm = options?.llm ?? new OpenAI();
|
||||
this.nodes = options?.nodes ?? 5;
|
||||
|
||||
this.nodeTemplate = options?.nodeTemplate
|
||||
? new PromptTemplate({
|
||||
templateVars: ["context"],
|
||||
template: options.nodeTemplate,
|
||||
})
|
||||
: defaultTitleExtractorPromptTemplate;
|
||||
|
||||
this.combineTemplate = options?.combineTemplate
|
||||
? new PromptTemplate({
|
||||
templateVars: ["context"],
|
||||
template: options.combineTemplate,
|
||||
})
|
||||
: defaultTitleCombinePromptTemplate;
|
||||
this.nodeTemplate =
|
||||
options?.nodeTemplate ?? defaultTitleExtractorPromptTemplate();
|
||||
this.combineTemplate =
|
||||
options?.combineTemplate ?? defaultTitleCombinePromptTemplate();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -246,8 +218,8 @@ export class TitleExtractor extends BaseExtractor {
|
||||
const titleCandidates = await this.getTitlesCandidates(nodes);
|
||||
const combinedTitles = titleCandidates.join(", ");
|
||||
const completion = await this.llm.complete({
|
||||
prompt: this.combineTemplate.format({
|
||||
context: combinedTitles,
|
||||
prompt: defaultTitleCombinePromptTemplate({
|
||||
contextStr: combinedTitles,
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -260,8 +232,8 @@ export class TitleExtractor extends BaseExtractor {
|
||||
private async getTitlesCandidates(nodes: BaseNode[]): Promise<string[]> {
|
||||
const titleJobs = nodes.map(async (node) => {
|
||||
const completion = await this.llm.complete({
|
||||
prompt: this.nodeTemplate.format({
|
||||
context: node.getContent(MetadataMode.ALL),
|
||||
prompt: defaultTitleExtractorPromptTemplate({
|
||||
contextStr: node.getContent(MetadataMode.ALL),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -275,7 +247,7 @@ export class TitleExtractor extends BaseExtractor {
|
||||
type QuestionAnswerExtractArgs = {
|
||||
llm?: LLM;
|
||||
questions?: number;
|
||||
promptTemplate?: QuestionExtractPrompt["template"];
|
||||
promptTemplate?: string;
|
||||
embeddingOnly?: boolean;
|
||||
};
|
||||
|
||||
@@ -304,7 +276,7 @@ export class QuestionsAnsweredExtractor extends BaseExtractor {
|
||||
* The prompt template to use for the question extractor.
|
||||
* @type {string}
|
||||
*/
|
||||
promptTemplate: QuestionExtractPrompt;
|
||||
promptTemplate: string;
|
||||
|
||||
/**
|
||||
* Wheter to use metadata for embeddings only
|
||||
@@ -317,7 +289,7 @@ export class QuestionsAnsweredExtractor extends BaseExtractor {
|
||||
* Constructor for the QuestionsAnsweredExtractor class.
|
||||
* @param {LLM} llm LLM instance.
|
||||
* @param {number} questions Number of questions to generate.
|
||||
* @param {TextQAPrompt} promptTemplate The prompt template to use for the question extractor.
|
||||
* @param {string} promptTemplate The prompt template to use for the question extractor.
|
||||
* @param {boolean} embeddingOnly Wheter to use metadata for embeddings only.
|
||||
*/
|
||||
constructor(options?: QuestionAnswerExtractArgs) {
|
||||
@@ -328,14 +300,12 @@ export class QuestionsAnsweredExtractor extends BaseExtractor {
|
||||
|
||||
this.llm = options?.llm ?? new OpenAI();
|
||||
this.questions = options?.questions ?? 5;
|
||||
this.promptTemplate = options?.promptTemplate
|
||||
? new PromptTemplate({
|
||||
templateVars: ["numQuestions", "context"],
|
||||
template: options.promptTemplate,
|
||||
}).partialFormat({
|
||||
numQuestions: "5",
|
||||
})
|
||||
: defaultQuestionExtractPrompt;
|
||||
this.promptTemplate =
|
||||
options?.promptTemplate ??
|
||||
defaultQuestionAnswerPromptTemplate({
|
||||
numQuestions: this.questions,
|
||||
contextStr: "",
|
||||
});
|
||||
this.embeddingOnly = options?.embeddingOnly ?? false;
|
||||
}
|
||||
|
||||
@@ -353,9 +323,9 @@ export class QuestionsAnsweredExtractor extends BaseExtractor {
|
||||
|
||||
const contextStr = node.getContent(this.metadataMode);
|
||||
|
||||
const prompt = this.promptTemplate.format({
|
||||
context: contextStr,
|
||||
numQuestions: this.questions.toString(),
|
||||
const prompt = defaultQuestionAnswerPromptTemplate({
|
||||
contextStr,
|
||||
numQuestions: this.questions,
|
||||
});
|
||||
|
||||
const questions = await this.llm.complete({
|
||||
@@ -386,7 +356,7 @@ export class QuestionsAnsweredExtractor extends BaseExtractor {
|
||||
type SummaryExtractArgs = {
|
||||
llm?: LLM;
|
||||
summaries?: string[];
|
||||
promptTemplate?: SummaryPrompt["template"];
|
||||
promptTemplate?: string;
|
||||
};
|
||||
|
||||
type ExtractSummary = {
|
||||
@@ -415,7 +385,7 @@ export class SummaryExtractor extends BaseExtractor {
|
||||
* The prompt template to use for the summary extractor.
|
||||
* @type {string}
|
||||
*/
|
||||
promptTemplate: SummaryPrompt;
|
||||
promptTemplate: string;
|
||||
|
||||
private selfSummary: boolean;
|
||||
private prevSummary: boolean;
|
||||
@@ -434,12 +404,8 @@ export class SummaryExtractor extends BaseExtractor {
|
||||
|
||||
this.llm = options?.llm ?? new OpenAI();
|
||||
this.summaries = summaries;
|
||||
this.promptTemplate = options?.promptTemplate
|
||||
? new PromptTemplate({
|
||||
templateVars: ["context"],
|
||||
template: options.promptTemplate,
|
||||
})
|
||||
: defaultSummaryPrompt;
|
||||
this.promptTemplate =
|
||||
options?.promptTemplate ?? defaultSummaryExtractorPromptTemplate();
|
||||
|
||||
this.selfSummary = summaries?.includes("self") ?? false;
|
||||
this.prevSummary = summaries?.includes("prev") ?? false;
|
||||
@@ -456,10 +422,10 @@ export class SummaryExtractor extends BaseExtractor {
|
||||
return "";
|
||||
}
|
||||
|
||||
const context = node.getContent(this.metadataMode);
|
||||
const contextStr = node.getContent(this.metadataMode);
|
||||
|
||||
const prompt = this.promptTemplate.format({
|
||||
context,
|
||||
const prompt = defaultSummaryExtractorPromptTemplate({
|
||||
contextStr,
|
||||
});
|
||||
|
||||
const summary = await this.llm.complete({
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
export interface DefaultPromptTemplate {
|
||||
contextStr: string;
|
||||
}
|
||||
|
||||
export interface DefaultKeywordExtractorPromptTemplate
|
||||
extends DefaultPromptTemplate {
|
||||
keywords: number;
|
||||
}
|
||||
|
||||
export interface DefaultQuestionAnswerPromptTemplate
|
||||
extends DefaultPromptTemplate {
|
||||
numQuestions: number;
|
||||
}
|
||||
|
||||
export interface DefaultNodeTextTemplate {
|
||||
metadataStr: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export const defaultKeywordExtractorPromptTemplate = ({
|
||||
contextStr = "",
|
||||
keywords = 5,
|
||||
}: DefaultKeywordExtractorPromptTemplate) => `${contextStr}
|
||||
Give ${keywords} unique keywords for this document.
|
||||
Format as comma separated.
|
||||
Keywords: `;
|
||||
|
||||
export const defaultTitleExtractorPromptTemplate = (
|
||||
{ contextStr = "" }: DefaultPromptTemplate = {
|
||||
contextStr: "",
|
||||
},
|
||||
) => `${contextStr}
|
||||
Give a title that summarizes all of the unique entities, titles or themes found in the context.
|
||||
Title: `;
|
||||
|
||||
export const defaultTitleCombinePromptTemplate = (
|
||||
{ contextStr = "" }: DefaultPromptTemplate = {
|
||||
contextStr: "",
|
||||
},
|
||||
) => `${contextStr}
|
||||
Based on the above candidate titles and contents, what is the comprehensive title for this document?
|
||||
Title: `;
|
||||
|
||||
export const defaultQuestionAnswerPromptTemplate = (
|
||||
{ contextStr = "", numQuestions = 5 }: DefaultQuestionAnswerPromptTemplate = {
|
||||
contextStr: "",
|
||||
numQuestions: 5,
|
||||
},
|
||||
) => `${contextStr}
|
||||
Given the contextual informations, generate ${numQuestions} questions this context can provides specific answers to which are unlikely to be found else where. Higher-level summaries of surrounding context may be provideds as well.
|
||||
Try using these summaries to generate better questions that this context can answer.
|
||||
`;
|
||||
|
||||
export const defaultSummaryExtractorPromptTemplate = (
|
||||
{ contextStr = "" }: DefaultPromptTemplate = {
|
||||
contextStr: "",
|
||||
},
|
||||
) => `${contextStr}
|
||||
Summarize the key topics and entities of the sections.
|
||||
Summary: `;
|
||||
|
||||
export const defaultNodeTextTemplate = ({
|
||||
metadataStr = "",
|
||||
content = "",
|
||||
}: {
|
||||
metadataStr?: string;
|
||||
content?: string;
|
||||
} = {}) => `[Excerpt from document]
|
||||
${metadataStr}
|
||||
Excerpt:
|
||||
-----
|
||||
${content}
|
||||
-----
|
||||
`;
|
||||
@@ -1,10 +1,10 @@
|
||||
import { defaultNodeTextTemplate } from "@llamaindex/core/prompts";
|
||||
import {
|
||||
BaseNode,
|
||||
MetadataMode,
|
||||
TextNode,
|
||||
TransformComponent,
|
||||
} from "@llamaindex/core/schema";
|
||||
import { defaultNodeTextTemplate } from "./prompts.js";
|
||||
|
||||
/*
|
||||
* Abstract class for all extractors.
|
||||
@@ -71,7 +71,7 @@ export abstract class BaseExtractor extends TransformComponent {
|
||||
if (newNodes[idx] instanceof TextNode) {
|
||||
newNodes[idx] = new TextNode({
|
||||
...newNodes[idx],
|
||||
textTemplate: defaultNodeTextTemplate.format(),
|
||||
textTemplate: defaultNodeTextTemplate(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,8 @@ import type { StorageContext } from "../../storage/StorageContext.js";
|
||||
import { storageContextFromDefaults } from "../../storage/StorageContext.js";
|
||||
import type { BaseIndexStore } from "../../storage/indexStore/types.js";
|
||||
import type {
|
||||
BaseVectorStore,
|
||||
MetadataFilters,
|
||||
VectorStore,
|
||||
VectorStoreByType,
|
||||
VectorStoreQueryResult,
|
||||
} from "../../vector-store/index.js";
|
||||
@@ -264,7 +264,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
|
||||
}
|
||||
|
||||
static async fromVectorStore(
|
||||
vectorStore: BaseVectorStore,
|
||||
vectorStore: VectorStore,
|
||||
serviceContext?: ServiceContext,
|
||||
) {
|
||||
return this.fromVectorStores(
|
||||
@@ -307,7 +307,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
|
||||
protected async insertNodesToStore(
|
||||
newIds: string[],
|
||||
nodes: BaseNode[],
|
||||
vectorStore: BaseVectorStore,
|
||||
vectorStore: VectorStore,
|
||||
): Promise<void> {
|
||||
// NOTE: if the vector store doesn't store text,
|
||||
// we need to add the nodes to the index struct and document store
|
||||
@@ -357,7 +357,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
|
||||
}
|
||||
|
||||
protected async deleteRefDocFromStore(
|
||||
vectorStore: BaseVectorStore,
|
||||
vectorStore: VectorStore,
|
||||
refDocId: string,
|
||||
): Promise<void> {
|
||||
await vectorStore.delete(refDocId);
|
||||
@@ -425,7 +425,7 @@ export class VectorIndexRetriever extends BaseRetriever {
|
||||
let nodesWithScores: NodeWithScore[] = [];
|
||||
|
||||
for (const type in vectorStores) {
|
||||
const vectorStore: BaseVectorStore = vectorStores[type as ModalityType]!;
|
||||
const vectorStore: VectorStore = vectorStores[type as ModalityType]!;
|
||||
nodesWithScores = nodesWithScores.concat(
|
||||
await this.retrieveQuery(query, type as ModalityType, vectorStore),
|
||||
);
|
||||
@@ -436,7 +436,7 @@ export class VectorIndexRetriever extends BaseRetriever {
|
||||
protected async retrieveQuery(
|
||||
query: MessageContent,
|
||||
type: ModalityType,
|
||||
vectorStore: BaseVectorStore,
|
||||
vectorStore: VectorStore,
|
||||
filters?: MetadataFilters,
|
||||
): Promise<NodeWithScore[]> {
|
||||
// convert string message to multi-modal format
|
||||
|
||||
@@ -7,10 +7,7 @@ import {
|
||||
type Metadata,
|
||||
} from "@llamaindex/core/schema";
|
||||
import type { BaseDocumentStore } from "../storage/docStore/types.js";
|
||||
import type {
|
||||
BaseVectorStore,
|
||||
VectorStoreByType,
|
||||
} from "../vector-store/types.js";
|
||||
import type { VectorStore, VectorStoreByType } from "../vector-store/types.js";
|
||||
import { IngestionCache, getTransformationHash } from "./IngestionCache.js";
|
||||
import {
|
||||
DocStoreStrategy,
|
||||
@@ -62,7 +59,7 @@ export class IngestionPipeline {
|
||||
transformations: TransformComponent[] = [];
|
||||
documents?: Document[] | undefined;
|
||||
reader?: BaseReader | undefined;
|
||||
vectorStore?: BaseVectorStore | undefined;
|
||||
vectorStore?: VectorStore | undefined;
|
||||
vectorStores?: VectorStoreByType | undefined;
|
||||
docStore?: BaseDocumentStore;
|
||||
docStoreStrategy: DocStoreStrategy = DocStoreStrategy.UPSERTS;
|
||||
@@ -136,7 +133,7 @@ export async function addNodesToVectorStores(
|
||||
nodesAdded?: (
|
||||
newIds: string[],
|
||||
nodes: BaseNode<Metadata>[],
|
||||
vectorStore: BaseVectorStore,
|
||||
vectorStore: VectorStore,
|
||||
) => Promise<void>,
|
||||
) {
|
||||
const nodeMap = splitNodesByType(nodes);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseNode, TransformComponent } from "@llamaindex/core/schema";
|
||||
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
|
||||
import type { BaseVectorStore } from "../../vector-store/types.js";
|
||||
import type { VectorStore } from "../../vector-store/types.js";
|
||||
import { classify } from "./classify.js";
|
||||
|
||||
/**
|
||||
@@ -9,9 +9,9 @@ import { classify } from "./classify.js";
|
||||
*/
|
||||
export class UpsertsAndDeleteStrategy extends TransformComponent {
|
||||
protected docStore: BaseDocumentStore;
|
||||
protected vectorStores: BaseVectorStore[] | undefined;
|
||||
protected vectorStores: VectorStore[] | undefined;
|
||||
|
||||
constructor(docStore: BaseDocumentStore, vectorStores?: BaseVectorStore[]) {
|
||||
constructor(docStore: BaseDocumentStore, vectorStores?: VectorStore[]) {
|
||||
super(async (nodes: BaseNode[]): Promise<BaseNode[]> => {
|
||||
const { dedupedNodes, missingDocs, unusedDocs } = await classify(
|
||||
this.docStore,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BaseNode, TransformComponent } from "@llamaindex/core/schema";
|
||||
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
|
||||
import type { BaseVectorStore } from "../../vector-store/types.js";
|
||||
import type { VectorStore } from "../../vector-store/types.js";
|
||||
import { classify } from "./classify.js";
|
||||
|
||||
/**
|
||||
@@ -8,9 +8,9 @@ import { classify } from "./classify.js";
|
||||
*/
|
||||
export class UpsertsStrategy extends TransformComponent {
|
||||
protected docStore: BaseDocumentStore;
|
||||
protected vectorStores: BaseVectorStore[] | undefined;
|
||||
protected vectorStores: VectorStore[] | undefined;
|
||||
|
||||
constructor(docStore: BaseDocumentStore, vectorStores?: BaseVectorStore[]) {
|
||||
constructor(docStore: BaseDocumentStore, vectorStores?: VectorStore[]) {
|
||||
super(async (nodes: BaseNode[]): Promise<BaseNode[]> => {
|
||||
const { dedupedNodes, unusedDocs } = await classify(this.docStore, nodes);
|
||||
// remove unused docs
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { TransformComponent } from "@llamaindex/core/schema";
|
||||
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
|
||||
import type { BaseVectorStore } from "../../vector-store/types.js";
|
||||
import type { VectorStore } from "../../vector-store/types.js";
|
||||
import { DuplicatesStrategy } from "./DuplicatesStrategy.js";
|
||||
import { UpsertsAndDeleteStrategy } from "./UpsertsAndDeleteStrategy.js";
|
||||
import { UpsertsStrategy } from "./UpsertsStrategy.js";
|
||||
@@ -28,7 +28,7 @@ class NoOpStrategy extends TransformComponent {
|
||||
export function createDocStoreStrategy(
|
||||
docStoreStrategy: DocStoreStrategy,
|
||||
docStore?: BaseDocumentStore,
|
||||
vectorStores: BaseVectorStore[] = [],
|
||||
vectorStores: VectorStore[] = [],
|
||||
): TransformComponent {
|
||||
if (docStoreStrategy === DocStoreStrategy.NONE) {
|
||||
return new NoOpStrategy();
|
||||
|
||||
@@ -44,15 +44,12 @@ import {
|
||||
export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
|
||||
[GEMINI_MODEL.GEMINI_PRO]: { contextWindow: 30720 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_VISION]: { contextWindow: 12288 },
|
||||
// multi-modal/multi turn
|
||||
[GEMINI_MODEL.GEMINI_PRO_LATEST]: { contextWindow: 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST]: { contextWindow: 10 ** 6 },
|
||||
// multi-modal/multi turn
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5]: { contextWindow: 2 * 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5_FLASH]: { contextWindow: 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5_LATEST]: { contextWindow: 2 * 10 ** 6 },
|
||||
[GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST]: { contextWindow: 10 ** 6 },
|
||||
};
|
||||
|
||||
const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
|
||||
@@ -62,10 +59,6 @@ const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
|
||||
GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
|
||||
GEMINI_MODEL.GEMINI_PRO_1_5,
|
||||
GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
|
||||
GEMINI_MODEL.GEMINI_PRO_LATEST,
|
||||
GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST,
|
||||
GEMINI_MODEL.GEMINI_PRO_1_5_LATEST,
|
||||
GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST,
|
||||
];
|
||||
|
||||
const DEFAULT_GEMINI_PARAMS = {
|
||||
|
||||
@@ -56,14 +56,10 @@ export enum GEMINI_MODEL {
|
||||
GEMINI_PRO = "gemini-pro",
|
||||
GEMINI_PRO_VISION = "gemini-pro-vision",
|
||||
GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
|
||||
GEMINI_PRO_FLASH_LATEST = "gemini-1.5-flash-latest",
|
||||
GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
|
||||
GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
|
||||
GEMINI_PRO_1_5 = "gemini-1.5-pro-001",
|
||||
GEMINI_PRO_1_5_FLASH = "gemini-1.5-flash-001",
|
||||
// Note: should be switched to -latest suffix when google supports it
|
||||
GEMINI_PRO_1_5_LATEST = "gemini-1.5-pro-002",
|
||||
GEMINI_PRO_1_5_FLASH_LATEST = "gemini-1.5-flash-002",
|
||||
}
|
||||
|
||||
export interface GeminiModelInfo {
|
||||
|
||||
@@ -37,14 +37,12 @@ export default function withLlamaIndex(config: any) {
|
||||
webpackConfig.resolve.alias = {
|
||||
...webpackConfig.resolve.alias,
|
||||
"@google-cloud/vertexai": false,
|
||||
unpdf: false,
|
||||
};
|
||||
// Following lines will fix issues with onnxruntime-node when using pnpm
|
||||
// See: https://github.com/vercel/next.js/issues/43433
|
||||
webpackConfig.externals.push({
|
||||
"onnxruntime-node": "commonjs onnxruntime-node",
|
||||
sharp: "commonjs sharp",
|
||||
chromadb: "commonjs chromadb",
|
||||
});
|
||||
return webpackConfig;
|
||||
};
|
||||
|
||||
@@ -7,10 +7,7 @@ import { path } from "@llamaindex/env";
|
||||
import { getImageEmbedModel } from "../internal/settings/image-embed-model.js";
|
||||
import type { ServiceContext } from "../ServiceContext.js";
|
||||
import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js";
|
||||
import type {
|
||||
BaseVectorStore,
|
||||
VectorStoreByType,
|
||||
} from "../vector-store/types.js";
|
||||
import type { VectorStore, VectorStoreByType } from "../vector-store/types.js";
|
||||
import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
|
||||
import type { BaseDocumentStore } from "./docStore/types.js";
|
||||
import { SimpleIndexStore } from "./indexStore/SimpleIndexStore.js";
|
||||
@@ -25,7 +22,7 @@ export interface StorageContext {
|
||||
type BuilderParams = {
|
||||
docStore: BaseDocumentStore;
|
||||
indexStore: BaseIndexStore;
|
||||
vectorStore: BaseVectorStore;
|
||||
vectorStore: VectorStore;
|
||||
vectorStores: VectorStoreByType;
|
||||
storeImages: boolean;
|
||||
persistDir: string;
|
||||
@@ -53,7 +50,7 @@ export async function storageContextFromDefaults({
|
||||
}
|
||||
if (storeImages && !(ModalityType.IMAGE in vectorStores)) {
|
||||
vectorStores[ModalityType.IMAGE] = new SimpleVectorStore({
|
||||
embeddingModel: new (await getImageEmbedModel())(),
|
||||
embedModel: new (await getImageEmbedModel())(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -2,30 +2,24 @@ import {
|
||||
Collection,
|
||||
DataAPIClient,
|
||||
Db,
|
||||
type Filter,
|
||||
type FindOptions,
|
||||
type SomeDoc,
|
||||
} from "@datastax/astra-db-ts";
|
||||
import type { BaseNode } from "@llamaindex/core/schema";
|
||||
import { MetadataMode } from "@llamaindex/core/schema";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
FilterOperator,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
VectorStoreBase,
|
||||
type IEmbedModel,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
import {
|
||||
metadataDictToNode,
|
||||
nodeToMetadata,
|
||||
parseArrayValue,
|
||||
} from "./utils.js";
|
||||
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
|
||||
|
||||
export class AstraDBVectorStore extends BaseVectorStore {
|
||||
export class AstraDBVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
flatMetadata: boolean = true;
|
||||
|
||||
@@ -43,9 +37,9 @@ export class AstraDBVectorStore extends BaseVectorStore {
|
||||
endpoint: string;
|
||||
namespace?: string;
|
||||
};
|
||||
} & VectorStoreBaseParams,
|
||||
} & Partial<IEmbedModel>,
|
||||
) {
|
||||
super(init);
|
||||
super(init?.embedModel);
|
||||
const token = init?.params?.token ?? getEnv("ASTRA_DB_APPLICATION_TOKEN");
|
||||
const endpoint = init?.params?.endpoint ?? getEnv("ASTRA_DB_API_ENDPOINT");
|
||||
|
||||
@@ -189,8 +183,12 @@ export class AstraDBVectorStore extends BaseVectorStore {
|
||||
}
|
||||
const collection = this.collection;
|
||||
|
||||
const astraFilter = this.toAstraFilter(query.filters);
|
||||
const cursor = await collection.find(astraFilter, <FindOptions>{
|
||||
const filters: Record<string, any> = {};
|
||||
query.filters?.filters?.forEach((f) => {
|
||||
filters[f.key] = f.value;
|
||||
});
|
||||
|
||||
const cursor = await collection.find(filters, <FindOptions>{
|
||||
...options,
|
||||
sort: query.queryEmbedding
|
||||
? { $vector: query.queryEmbedding }
|
||||
@@ -232,39 +230,4 @@ export class AstraDBVectorStore extends BaseVectorStore {
|
||||
nodes,
|
||||
};
|
||||
}
|
||||
|
||||
private toAstraFilter(filters?: MetadataFilters): Filter<SomeDoc> {
|
||||
if (!filters || filters.filters?.length === 0) return {};
|
||||
const condition = filters.condition ?? FilterCondition.AND;
|
||||
const listFilter = filters.filters.map((f) => this.buildFilterItem(f));
|
||||
if (condition === FilterCondition.OR) return { $or: listFilter };
|
||||
if (condition === FilterCondition.AND) return { $and: listFilter };
|
||||
throw new Error(`Not supported filter condition: ${condition}`);
|
||||
}
|
||||
|
||||
private buildFilterItem(filter: MetadataFilter): Filter<SomeDoc> {
|
||||
const { key, operator, value } = filter;
|
||||
switch (operator) {
|
||||
case FilterOperator.EQ:
|
||||
return { [key]: value };
|
||||
case FilterOperator.NE:
|
||||
return { [key]: { $ne: value } };
|
||||
case FilterOperator.GT:
|
||||
return { [key]: { $gt: value } };
|
||||
case FilterOperator.LT:
|
||||
return { [key]: { $lt: value } };
|
||||
case FilterOperator.GTE:
|
||||
return { [key]: { $gte: value } };
|
||||
case FilterOperator.LTE:
|
||||
return { [key]: { $lte: value } };
|
||||
case FilterOperator.IN:
|
||||
return { [key]: { $in: parseArrayValue(value) } };
|
||||
case FilterOperator.NIN:
|
||||
return { [key]: { $nin: parseArrayValue(value) } };
|
||||
case FilterOperator.IS_EMPTY:
|
||||
return { [key]: { $size: 0 } };
|
||||
default:
|
||||
throw new Error(`Not supported filter operator: ${operator}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,20 +2,21 @@ import type { BaseNode } from "@llamaindex/core/schema";
|
||||
import { MetadataMode } from "@llamaindex/core/schema";
|
||||
import {
|
||||
ChromaClient,
|
||||
IncludeEnum,
|
||||
type AddParams,
|
||||
type ChromaClientParams,
|
||||
type Collection,
|
||||
type DeleteParams,
|
||||
type QueryRecordsParams,
|
||||
type QueryParams,
|
||||
type QueryResponse,
|
||||
type Where,
|
||||
type WhereDocument,
|
||||
} from "chromadb";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
FilterOperator,
|
||||
VectorStoreBase,
|
||||
VectorStoreQueryMode,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type IEmbedModel,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -30,22 +31,12 @@ type ChromaQueryOptions = {
|
||||
whereDocument?: WhereDocument;
|
||||
};
|
||||
|
||||
type Collection = Awaited<ReturnType<ChromaClient["getOrCreateCollection"]>>;
|
||||
|
||||
const DEFAULT_TEXT_KEY = "text";
|
||||
|
||||
type ChromaFilterCondition = "$and" | "$or";
|
||||
type ChromaFilterOperator =
|
||||
| "$eq"
|
||||
| "$ne"
|
||||
| "$gt"
|
||||
| "$lt"
|
||||
| "$gte"
|
||||
| "$lte"
|
||||
| "$in"
|
||||
| "$nin";
|
||||
|
||||
export class ChromaVectorStore extends BaseVectorStore {
|
||||
export class ChromaVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
flatMetadata: boolean = true;
|
||||
textKey: string;
|
||||
@@ -58,9 +49,9 @@ export class ChromaVectorStore extends BaseVectorStore {
|
||||
collectionName: string;
|
||||
textKey?: string;
|
||||
chromaClientParams?: ChromaClientParams;
|
||||
} & VectorStoreBaseParams,
|
||||
} & Partial<IEmbedModel>,
|
||||
) {
|
||||
super(init);
|
||||
super(init.embedModel);
|
||||
this.collectionName = init.collectionName;
|
||||
this.chromaClient = new ChromaClient(init.chromaClientParams);
|
||||
this.textKey = init.textKey ?? DEFAULT_TEXT_KEY;
|
||||
@@ -80,7 +71,7 @@ export class ChromaVectorStore extends BaseVectorStore {
|
||||
return this.collection;
|
||||
}
|
||||
|
||||
private getDataToInsert(nodes: BaseNode[]) {
|
||||
private getDataToInsert(nodes: BaseNode[]): AddParams {
|
||||
const metadatas = nodes.map((node) =>
|
||||
nodeToMetadata(node, true, this.textKey, this.flatMetadata),
|
||||
);
|
||||
@@ -115,79 +106,6 @@ export class ChromaVectorStore extends BaseVectorStore {
|
||||
});
|
||||
}
|
||||
|
||||
private transformChromaFilterCondition(
|
||||
condition: FilterCondition,
|
||||
): ChromaFilterCondition {
|
||||
switch (condition) {
|
||||
case FilterCondition.AND:
|
||||
return "$and";
|
||||
case FilterCondition.OR:
|
||||
return "$or";
|
||||
default:
|
||||
throw new Error(`Filter condition ${condition} not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
private transformChromaFilterOperator(
|
||||
operator: FilterOperator,
|
||||
): ChromaFilterOperator {
|
||||
switch (operator) {
|
||||
case FilterOperator.EQ:
|
||||
return "$eq";
|
||||
case FilterOperator.NE:
|
||||
return "$ne";
|
||||
case FilterOperator.GT:
|
||||
return "$gt";
|
||||
case FilterOperator.LT:
|
||||
return "$lt";
|
||||
case FilterOperator.GTE:
|
||||
return "$gte";
|
||||
case FilterOperator.LTE:
|
||||
return "$lte";
|
||||
case FilterOperator.IN:
|
||||
return "$in";
|
||||
case FilterOperator.NIN:
|
||||
return "$nin";
|
||||
default:
|
||||
throw new Error(`Filter operator ${operator} not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
private toChromaFilter(filters: MetadataFilters): Where {
|
||||
const chromaFilter: Where = {};
|
||||
const filtersList: Where[] = [];
|
||||
|
||||
const condition = filters.condition
|
||||
? this.transformChromaFilterCondition(
|
||||
filters.condition as FilterCondition,
|
||||
)
|
||||
: "$and";
|
||||
|
||||
if (filters.filters) {
|
||||
for (const filter of filters.filters) {
|
||||
if (filter.operator) {
|
||||
filtersList.push({
|
||||
[filter.key]: {
|
||||
[this.transformChromaFilterOperator(
|
||||
filter.operator as FilterOperator,
|
||||
)]: filter.value,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
filtersList.push({ [filter.key]: filter.value });
|
||||
}
|
||||
}
|
||||
|
||||
if (filtersList.length === 1) {
|
||||
return filtersList[0]!;
|
||||
} else if (filtersList.length > 1) {
|
||||
chromaFilter[condition] = filtersList;
|
||||
}
|
||||
}
|
||||
|
||||
return chromaFilter;
|
||||
}
|
||||
|
||||
async query(
|
||||
query: VectorStoreQuery,
|
||||
options?: ChromaQueryOptions,
|
||||
@@ -199,22 +117,49 @@ export class ChromaVectorStore extends BaseVectorStore {
|
||||
throw new Error("ChromaDB does not support querying by mode");
|
||||
}
|
||||
|
||||
let chromaWhere: Where = {};
|
||||
if (query.filters) {
|
||||
chromaWhere = this.toChromaFilter(query.filters);
|
||||
// fixme: type is broken
|
||||
let chromaWhere: any = {};
|
||||
if (query.filters?.filters) {
|
||||
query.filters.filters.map((filter) => {
|
||||
const filterKey = filter.key;
|
||||
const filterValue = filter.value;
|
||||
if (filterKey in chromaWhere || "$or" in chromaWhere) {
|
||||
if (!chromaWhere["$or"]) {
|
||||
chromaWhere = {
|
||||
$or: [
|
||||
{
|
||||
...chromaWhere,
|
||||
},
|
||||
{
|
||||
[filterKey]: filterValue,
|
||||
},
|
||||
],
|
||||
};
|
||||
} else {
|
||||
chromaWhere["$or"].push({
|
||||
[filterKey]: filterValue,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
chromaWhere[filterKey] = filterValue;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const collection = await this.getCollection();
|
||||
const queryResponse: QueryResponse = await collection.query(<
|
||||
QueryRecordsParams
|
||||
>{
|
||||
const queryResponse: QueryResponse = await collection.query(<QueryParams>{
|
||||
queryEmbeddings: query.queryEmbedding ?? undefined,
|
||||
queryTexts: query.queryStr ?? undefined,
|
||||
nResults: query.similarityTopK,
|
||||
where: Object.keys(chromaWhere).length ? chromaWhere : undefined,
|
||||
whereDocument: options?.whereDocument,
|
||||
//ChromaDB doesn't return the result embeddings by default so we need to include them
|
||||
include: ["distances", "metadatas", "documents", "embeddings"],
|
||||
include: [
|
||||
IncludeEnum.Distances,
|
||||
IncludeEnum.Metadatas,
|
||||
IncludeEnum.Documents,
|
||||
IncludeEnum.Embeddings,
|
||||
],
|
||||
});
|
||||
|
||||
const vectorStoreQueryResult: VectorStoreQueryResult = {
|
||||
|
||||
@@ -10,9 +10,10 @@ import {
|
||||
type SearchSimpleReq,
|
||||
} from "@zilliz/milvus2-sdk-node";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
VectorStoreBase,
|
||||
type IEmbedModel,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -71,9 +72,12 @@ function parseScalarFilters(scalarFilters: MetadataFilters): string {
|
||||
return filters.join(` ${condition} `);
|
||||
}
|
||||
|
||||
export class MilvusVectorStore extends BaseVectorStore {
|
||||
export class MilvusVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
public storesText: boolean = true;
|
||||
public isEmbeddingQuery?: boolean = false;
|
||||
public isEmbeddingQuery?: boolean;
|
||||
private flatMetadata: boolean = true;
|
||||
|
||||
private milvusClient: MilvusClient;
|
||||
@@ -87,7 +91,7 @@ export class MilvusVectorStore extends BaseVectorStore {
|
||||
|
||||
constructor(
|
||||
init?: Partial<{ milvusClient: MilvusClient }> &
|
||||
VectorStoreBaseParams & {
|
||||
Partial<IEmbedModel> & {
|
||||
params?: {
|
||||
configOrAddress: ClientConfig | string;
|
||||
ssl?: boolean;
|
||||
@@ -102,7 +106,7 @@ export class MilvusVectorStore extends BaseVectorStore {
|
||||
embeddingKey?: string;
|
||||
},
|
||||
) {
|
||||
super(init);
|
||||
super(init?.embedModel);
|
||||
if (init?.milvusClient) {
|
||||
this.milvusClient = init.milvusClient;
|
||||
} else {
|
||||
|
||||
@@ -5,12 +5,12 @@ import { getEnv } from "@llamaindex/env";
|
||||
import type { BulkWriteOptions, Collection } from "mongodb";
|
||||
import { MongoClient } from "mongodb";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
VectorStoreBase,
|
||||
type FilterOperator,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -67,7 +67,10 @@ function toMongoDBFilter(filters?: MetadataFilters): Record<string, any> {
|
||||
* Vector store that uses MongoDB Atlas for storage and vector search.
|
||||
* This store uses the $vectorSearch aggregation stage to perform vector similarity search.
|
||||
*/
|
||||
export class MongoDBAtlasVectorSearch extends BaseVectorStore {
|
||||
export class MongoDBAtlasVectorSearch
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
flatMetadata: boolean = true;
|
||||
|
||||
@@ -144,9 +147,9 @@ export class MongoDBAtlasVectorSearch extends BaseVectorStore {
|
||||
autoCreateIndex?: boolean;
|
||||
indexedMetadataFields?: string[];
|
||||
embeddingDefinition?: Record<string, unknown>;
|
||||
} & VectorStoreBaseParams,
|
||||
},
|
||||
) {
|
||||
super(init);
|
||||
super(init.embedModel);
|
||||
if (init.mongodbClient) {
|
||||
this.mongodbClient = init.mongodbClient;
|
||||
} else {
|
||||
@@ -271,7 +274,7 @@ export class MongoDBAtlasVectorSearch extends BaseVectorStore {
|
||||
);
|
||||
}
|
||||
|
||||
client(): any {
|
||||
get client(): any {
|
||||
return this.mongodbClient;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,12 +4,12 @@ import type { IsomorphicDB } from "@llamaindex/core/vector-store";
|
||||
import type { VercelPool } from "@vercel/postgres";
|
||||
import type { Sql } from "postgres";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
FilterOperator,
|
||||
type MetadataFilter,
|
||||
type MetadataFilterValue,
|
||||
type VectorStoreBaseParams,
|
||||
VectorStoreBase,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -116,8 +116,7 @@ type PGVectorStoreBaseConfig = {
|
||||
embedModel?: BaseEmbedding | undefined;
|
||||
};
|
||||
|
||||
export type PGVectorStoreConfig = VectorStoreBaseParams &
|
||||
PGVectorStoreBaseConfig &
|
||||
export type PGVectorStoreConfig = PGVectorStoreBaseConfig &
|
||||
(
|
||||
| {
|
||||
/**
|
||||
@@ -148,7 +147,10 @@ export type PGVectorStoreConfig = VectorStoreBaseParams &
|
||||
* Provides support for writing and querying vector data in Postgres.
|
||||
* Note: Can't be used with data created using the Python version of the vector store (https://docs.llamaindex.ai/en/stable/examples/vector_stores/postgres/)
|
||||
*/
|
||||
export class PGVectorStore extends BaseVectorStore {
|
||||
export class PGVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
|
||||
private collection: string = DEFAULT_COLLECTION;
|
||||
@@ -161,7 +163,7 @@ export class PGVectorStore extends BaseVectorStore {
|
||||
private readonly clientConfig: pg.ClientConfig | null = null;
|
||||
|
||||
constructor(config: PGVectorStoreConfig) {
|
||||
super(config);
|
||||
super(config?.embedModel);
|
||||
this.schemaName = config?.schemaName ?? PGVECTOR_SCHEMA;
|
||||
this.tableName = config?.tableName ?? PGVECTOR_TABLE;
|
||||
this.dimensions = config?.dimensions ?? DEFAULT_DIMENSIONS;
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
FilterOperator,
|
||||
VectorStoreBase,
|
||||
type IEmbedModel,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -24,12 +25,15 @@ type PineconeParams = {
|
||||
chunkSize?: number;
|
||||
namespace?: string;
|
||||
textKey?: string;
|
||||
} & VectorStoreBaseParams;
|
||||
} & IEmbedModel;
|
||||
|
||||
/**
|
||||
* Provides support for writing and querying vector data in Pinecone.
|
||||
*/
|
||||
export class PineconeVectorStore extends BaseVectorStore {
|
||||
export class PineconeVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
|
||||
/*
|
||||
@@ -47,7 +51,7 @@ export class PineconeVectorStore extends BaseVectorStore {
|
||||
textKey: string;
|
||||
|
||||
constructor(params?: PineconeParams) {
|
||||
super(params);
|
||||
super(params?.embedModel);
|
||||
this.indexName =
|
||||
params?.indexName ?? getEnv("PINECONE_INDEX_NAME") ?? "llama";
|
||||
this.namespace = params?.namespace ?? getEnv("PINECONE_NAMESPACE") ?? "";
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import type { BaseNode } from "@llamaindex/core/schema";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterCondition,
|
||||
FilterOperator,
|
||||
VectorStoreBase,
|
||||
type IEmbedModel,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
|
||||
import { UUIDFromString } from "@llamaindex/core/utils";
|
||||
import type { QdrantClientParams, Schemas } from "@qdrant/js-client-rest";
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
|
||||
@@ -28,7 +30,7 @@ type QdrantParams = {
|
||||
url?: string;
|
||||
apiKey?: string;
|
||||
batchSize?: number;
|
||||
} & VectorStoreBaseParams;
|
||||
} & Partial<IEmbedModel>;
|
||||
|
||||
type QuerySearchResult = {
|
||||
id: string;
|
||||
@@ -41,7 +43,10 @@ type QuerySearchResult = {
|
||||
/**
|
||||
* Qdrant vector store.
|
||||
*/
|
||||
export class QdrantVectorStore extends BaseVectorStore {
|
||||
export class QdrantVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
|
||||
batchSize: number;
|
||||
@@ -65,9 +70,9 @@ export class QdrantVectorStore extends BaseVectorStore {
|
||||
url,
|
||||
apiKey,
|
||||
batchSize,
|
||||
...init
|
||||
embedModel,
|
||||
}: QdrantParams) {
|
||||
super(init);
|
||||
super(embedModel);
|
||||
if (!client && !url) {
|
||||
if (!url) {
|
||||
throw new Error("QdrantVectorStore requires url and collectionName");
|
||||
@@ -166,7 +171,7 @@ export class QdrantVectorStore extends BaseVectorStore {
|
||||
|
||||
for (let k = 0; k < nodeIds.length; k++) {
|
||||
const point: PointStruct = {
|
||||
id: nodeIds[k]!.id_,
|
||||
id: UUIDFromString(nodeIds[k]!.id_),
|
||||
payload: payloads[k]!,
|
||||
vector: vectors[k]!,
|
||||
};
|
||||
|
||||
@@ -5,12 +5,12 @@ import { fs, path } from "@llamaindex/env";
|
||||
import { getTopKEmbeddings, getTopKMMREmbeddings } from "../internal/utils.js";
|
||||
import { exists } from "../storage/FileSystem.js";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
FilterOperator,
|
||||
VectorStoreBase,
|
||||
VectorStoreQueryMode,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -120,17 +120,19 @@ class SimpleVectorStoreData {
|
||||
metadataDict: Record<string, MetadataValue> = {};
|
||||
}
|
||||
|
||||
export class SimpleVectorStore extends BaseVectorStore {
|
||||
export class SimpleVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = false;
|
||||
private data: SimpleVectorStoreData;
|
||||
private persistPath: string | undefined;
|
||||
|
||||
constructor(
|
||||
init?: {
|
||||
data?: SimpleVectorStoreData | undefined;
|
||||
} & VectorStoreBaseParams,
|
||||
) {
|
||||
super(init);
|
||||
constructor(init?: {
|
||||
data?: SimpleVectorStoreData | undefined;
|
||||
embedModel?: BaseEmbedding | undefined;
|
||||
}) {
|
||||
super(init?.embedModel);
|
||||
this.data = init?.data || new SimpleVectorStoreData();
|
||||
}
|
||||
|
||||
@@ -142,7 +144,7 @@ export class SimpleVectorStore extends BaseVectorStore {
|
||||
return await SimpleVectorStore.fromPersistPath(persistPath, embedModel);
|
||||
}
|
||||
|
||||
client(): any {
|
||||
get client(): any {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -270,7 +272,7 @@ export class SimpleVectorStore extends BaseVectorStore {
|
||||
|
||||
static async fromPersistPath(
|
||||
persistPath: string,
|
||||
embeddingModel?: BaseEmbedding,
|
||||
embedModel?: BaseEmbedding,
|
||||
): Promise<SimpleVectorStore> {
|
||||
const dirPath = path.dirname(persistPath);
|
||||
if (!(await exists(dirPath))) {
|
||||
@@ -296,20 +298,20 @@ export class SimpleVectorStore extends BaseVectorStore {
|
||||
data.embeddingDict = dataDict.embeddingDict ?? {};
|
||||
data.textIdToRefDocId = dataDict.textIdToRefDocId ?? {};
|
||||
data.metadataDict = dataDict.metadataDict ?? {};
|
||||
const store = new SimpleVectorStore({ data, embeddingModel });
|
||||
const store = new SimpleVectorStore({ data, embedModel });
|
||||
store.persistPath = persistPath;
|
||||
return store;
|
||||
}
|
||||
|
||||
static fromDict(
|
||||
saveDict: SimpleVectorStoreData,
|
||||
embeddingModel?: BaseEmbedding,
|
||||
embedModel?: BaseEmbedding,
|
||||
): SimpleVectorStore {
|
||||
const data = new SimpleVectorStoreData();
|
||||
data.embeddingDict = saveDict.embeddingDict;
|
||||
data.textIdToRefDocId = saveDict.textIdToRefDocId;
|
||||
data.metadataDict = saveDict.metadataDict;
|
||||
return new SimpleVectorStore({ data, embeddingModel });
|
||||
return new SimpleVectorStore({ data, embedModel });
|
||||
}
|
||||
|
||||
toDict(): SimpleVectorStoreData {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import {
|
||||
BaseVectorStore,
|
||||
VectorStoreBase,
|
||||
type IEmbedModel,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -17,12 +18,15 @@ type UpstashParams = {
|
||||
token?: string;
|
||||
endpoint?: string;
|
||||
maxBatchSize?: number;
|
||||
} & VectorStoreBaseParams;
|
||||
} & IEmbedModel;
|
||||
|
||||
/**
|
||||
* Provides support for writing and querying vector data in Upstash.
|
||||
*/
|
||||
export class UpstashVectorStore extends BaseVectorStore {
|
||||
export class UpstashVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
storesText: boolean = true;
|
||||
|
||||
private db: Index;
|
||||
@@ -41,7 +45,7 @@ export class UpstashVectorStore extends BaseVectorStore {
|
||||
* ```
|
||||
*/
|
||||
constructor(params?: UpstashParams) {
|
||||
super(params);
|
||||
super(params?.embedModel);
|
||||
this.namespace = params?.namespace ?? "";
|
||||
this.maxBatchSize = params?.maxBatchSize ?? 1000;
|
||||
const token = params?.token ?? getEnv("UPSTASH_VECTOR_REST_TOKEN");
|
||||
|
||||
@@ -11,11 +11,12 @@ import weaviate, {
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { BaseHybridOptions } from "weaviate-client";
|
||||
import {
|
||||
BaseVectorStore,
|
||||
VectorStoreBase,
|
||||
VectorStoreQueryMode,
|
||||
type IEmbedModel,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreBaseParams,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
@@ -118,7 +119,10 @@ const toWeaviateFilter = (
|
||||
return Filters[condition](...filtersList);
|
||||
};
|
||||
|
||||
export class WeaviateVectorStore extends BaseVectorStore {
|
||||
export class WeaviateVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
public storesText: boolean = true;
|
||||
private flatMetadata: boolean = true;
|
||||
|
||||
@@ -133,7 +137,7 @@ export class WeaviateVectorStore extends BaseVectorStore {
|
||||
private metadataKey: string;
|
||||
|
||||
constructor(
|
||||
init?: VectorStoreBaseParams & {
|
||||
init?: Partial<IEmbedModel> & {
|
||||
weaviateClient?: WeaviateClient;
|
||||
cloudOptions?: {
|
||||
clusterURL?: string;
|
||||
@@ -146,7 +150,7 @@ export class WeaviateVectorStore extends BaseVectorStore {
|
||||
embeddingKey?: string;
|
||||
},
|
||||
) {
|
||||
super(init);
|
||||
super(init?.embedModel);
|
||||
|
||||
if (init?.weaviateClient) {
|
||||
// Use the provided client
|
||||
|
||||
@@ -76,28 +76,34 @@ export interface VectorStoreQuery {
|
||||
mmrThreshold?: number;
|
||||
}
|
||||
|
||||
// Supported types of vector stores (for each modality)
|
||||
export type VectorStoreByType = {
|
||||
[P in ModalityType]?: BaseVectorStore;
|
||||
};
|
||||
|
||||
export type VectorStoreBaseParams = {
|
||||
embeddingModel?: BaseEmbedding | undefined;
|
||||
};
|
||||
|
||||
export abstract class BaseVectorStore {
|
||||
embedModel: BaseEmbedding;
|
||||
abstract storesText: boolean;
|
||||
export interface VectorStoreNoEmbedModel {
|
||||
storesText: boolean;
|
||||
isEmbeddingQuery?: boolean;
|
||||
abstract client(): any;
|
||||
abstract add(embeddingResults: BaseNode[]): Promise<string[]>;
|
||||
abstract delete(refDocId: string, deleteOptions?: any): Promise<void>;
|
||||
abstract query(
|
||||
client(): any;
|
||||
add(embeddingResults: BaseNode[]): Promise<string[]>;
|
||||
delete(refDocId: string, deleteOptions?: any): Promise<void>;
|
||||
query(
|
||||
query: VectorStoreQuery,
|
||||
options?: any,
|
||||
): Promise<VectorStoreQueryResult>;
|
||||
}
|
||||
|
||||
protected constructor(params?: VectorStoreBaseParams) {
|
||||
this.embedModel = params?.embeddingModel ?? getEmbeddedModel();
|
||||
export interface IEmbedModel {
|
||||
embedModel: BaseEmbedding;
|
||||
}
|
||||
|
||||
export interface VectorStore extends VectorStoreNoEmbedModel, IEmbedModel {}
|
||||
|
||||
// Supported types of vector stores (for each modality)
|
||||
|
||||
export type VectorStoreByType = {
|
||||
[P in ModalityType]?: VectorStore;
|
||||
};
|
||||
|
||||
export abstract class VectorStoreBase implements IEmbedModel {
|
||||
embedModel: BaseEmbedding;
|
||||
|
||||
protected constructor(embedModel?: BaseEmbedding) {
|
||||
this.embedModel = embedModel ?? getEmbeddedModel();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,35 +102,6 @@ describe("[MetadataExtractor]: Extractors should populate the metadata", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("[MetadataExtractor] QuestionsAnsweredExtractor uses custom prompt template", async () => {
|
||||
const nodeParser = new SentenceSplitter();
|
||||
|
||||
const nodes = nodeParser.getNodesFromDocuments([
|
||||
new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
|
||||
]);
|
||||
|
||||
const llmCompleteSpy = vi.spyOn(serviceContext.llm, "complete");
|
||||
|
||||
const questionsAnsweredExtractor = new QuestionsAnsweredExtractor({
|
||||
llm: serviceContext.llm,
|
||||
questions: 5,
|
||||
promptTemplate: `This is a custom prompt template for {context} with {numQuestions} questions`,
|
||||
});
|
||||
|
||||
await questionsAnsweredExtractor.processNodes(nodes);
|
||||
|
||||
expect(llmCompleteSpy).toHaveBeenCalled();
|
||||
|
||||
// Build the expected prompt
|
||||
const expectedPrompt = `This is a custom prompt template for ${DEFAULT_LLM_TEXT_OUTPUT} with 5 questions`;
|
||||
|
||||
// Get the actual prompt used in llm.complete
|
||||
const actualPrompt = llmCompleteSpy.mock?.calls?.[0]?.[0];
|
||||
|
||||
// Assert that the prompts match
|
||||
expect(actualPrompt).toEqual({ prompt: expectedPrompt });
|
||||
});
|
||||
|
||||
test("[MetadataExtractor] SumamryExtractor returns sectionSummary metadata", async () => {
|
||||
const nodeParser = new SentenceSplitter();
|
||||
|
||||
@@ -148,33 +119,4 @@ describe("[MetadataExtractor]: Extractors should populate the metadata", () => {
|
||||
sectionSummary: DEFAULT_LLM_TEXT_OUTPUT,
|
||||
});
|
||||
});
|
||||
|
||||
test("[KeywordExtractor] KeywordExtractor uses custom prompt template", async () => {
|
||||
const nodeParser = new SentenceSplitter();
|
||||
|
||||
const nodes = nodeParser.getNodesFromDocuments([
|
||||
new Document({ text: DEFAULT_LLM_TEXT_OUTPUT }),
|
||||
]);
|
||||
|
||||
const llmCompleteSpy = vi.spyOn(serviceContext.llm, "complete");
|
||||
|
||||
const keywordExtractor = new KeywordExtractor({
|
||||
llm: serviceContext.llm,
|
||||
keywords: 5,
|
||||
promptTemplate: `This is a custom prompt template for {context} with {maxKeywords} keywords`,
|
||||
});
|
||||
|
||||
await keywordExtractor.processNodes(nodes);
|
||||
|
||||
expect(llmCompleteSpy).toHaveBeenCalled();
|
||||
|
||||
// Build the expected prompt
|
||||
const expectedPrompt = `This is a custom prompt template for ${DEFAULT_LLM_TEXT_OUTPUT} with 5 keywords`;
|
||||
|
||||
// Get the actual prompt used in llm.complete
|
||||
const actualPrompt = llmCompleteSpy.mock?.calls?.[0]?.[0];
|
||||
|
||||
// Assert that the prompts match
|
||||
expect(actualPrompt).toEqual({ prompt: expectedPrompt });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,7 +27,7 @@ describe("VectorStoreIndex", () => {
|
||||
runs: number = 2,
|
||||
): Promise<Array<number>> => {
|
||||
const documents = [new Document({ text: "lorem ipsem", id_: "1" })];
|
||||
const entries = [];
|
||||
const entries: number[] = [];
|
||||
for (let i = 0; i < runs; i++) {
|
||||
await VectorStoreIndex.fromDocuments(documents, {
|
||||
serviceContext,
|
||||
@@ -43,7 +43,7 @@ describe("VectorStoreIndex", () => {
|
||||
|
||||
test("fromDocuments stores duplicates without a doc store strategy", async () => {
|
||||
const entries = await testStrategy(DocStoreStrategy.NONE);
|
||||
expect(entries[0]! + 1).toBe(entries[1]);
|
||||
expect(entries[0]).toBe(entries[1]);
|
||||
});
|
||||
|
||||
test("fromDocuments ignores duplicates with upserts doc store strategy", async () => {
|
||||
|
||||
@@ -59,7 +59,7 @@ describe("SimpleVectorStore", () => {
|
||||
}),
|
||||
];
|
||||
store = new SimpleVectorStore({
|
||||
embeddingModel: {} as BaseEmbedding, // Mocking the embedModel
|
||||
embedModel: {} as BaseEmbedding, // Mocking the embedModel
|
||||
data: {
|
||||
embeddingDict: {},
|
||||
textIdToRefDocId: {},
|
||||
|
||||
@@ -1,24 +1,5 @@
|
||||
# @llamaindex/groq
|
||||
|
||||
## 0.0.16
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @llamaindex/openai@0.1.17
|
||||
|
||||
## 0.0.15
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [6a9a7b1]
|
||||
- @llamaindex/openai@0.1.16
|
||||
|
||||
## 0.0.14
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- @llamaindex/openai@0.1.15
|
||||
|
||||
## 0.0.13
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/groq",
|
||||
"description": "Groq Adapter for LlamaIndex",
|
||||
"version": "0.0.16",
|
||||
"version": "0.0.13",
|
||||
"type": "module",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
|
||||
@@ -1,20 +1,5 @@
|
||||
# @llamaindex/ollama
|
||||
|
||||
## 0.0.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- @llamaindex/core@0.3.0
|
||||
|
||||
## 0.0.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- @llamaindex/core@0.2.12
|
||||
|
||||
## 0.0.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/ollama",
|
||||
"description": "Ollama Adapter for LlamaIndex",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.6",
|
||||
"type": "module",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
|
||||
@@ -1,26 +1,5 @@
|
||||
# @llamaindex/openai
|
||||
|
||||
## 0.1.17
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1364e8e]
|
||||
- Updated dependencies [96fc69c]
|
||||
- @llamaindex/core@0.3.0
|
||||
|
||||
## 0.1.16
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 6a9a7b1: fix: take init api key into account
|
||||
|
||||
## 0.1.15
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [5f67820]
|
||||
- @llamaindex/core@0.2.12
|
||||
|
||||
## 0.1.14
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/openai",
|
||||
"description": "OpenAI Adapter for LlamaIndex",
|
||||
"version": "0.1.17",
|
||||
"version": "0.1.14",
|
||||
"type": "module",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
|
||||
@@ -114,8 +114,7 @@ export class OpenAIEmbedding extends BaseEmbedding {
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
this.apiKey =
|
||||
init?.session?.apiKey ?? init?.apiKey ?? getEnv("OPENAI_API_KEY");
|
||||
this.apiKey = init?.session?.apiKey ?? getEnv("OPENAI_API_KEY");
|
||||
this.lazySession = async () =>
|
||||
import("openai").then(({ OpenAI }) => {
|
||||
return (
|
||||
|
||||
Generated
+18
-64
@@ -139,7 +139,7 @@ importers:
|
||||
specifier: ^1.4.1
|
||||
version: 1.4.1
|
||||
'@llamaindex/core':
|
||||
specifier: ^0.3.0
|
||||
specifier: ^0.2.0
|
||||
version: link:../packages/core
|
||||
'@notionhq/client':
|
||||
specifier: ^2.2.15
|
||||
@@ -166,7 +166,7 @@ importers:
|
||||
specifier: ^1.0.14
|
||||
version: 1.0.14
|
||||
llamaindex:
|
||||
specifier: ^0.7.0
|
||||
specifier: ^0.6.0
|
||||
version: link:../packages/llamaindex
|
||||
mongodb:
|
||||
specifier: ^6.7.0
|
||||
@@ -592,8 +592,8 @@ importers:
|
||||
specifier: ^4.7.0
|
||||
version: 4.7.0(bufferutil@4.0.8)
|
||||
chromadb:
|
||||
specifier: 1.9.2
|
||||
version: 1.9.2(@google/generative-ai@0.12.0)(cohere-ai@7.13.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.650.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)(zod@3.23.8))
|
||||
specifier: 1.8.1
|
||||
version: 1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.13.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.650.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)(zod@3.23.8))
|
||||
cohere-ai:
|
||||
specifier: 7.13.0
|
||||
version: 7.13.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.650.0))(encoding@0.1.13)
|
||||
@@ -5574,21 +5574,6 @@ packages:
|
||||
openai:
|
||||
optional: true
|
||||
|
||||
chromadb@1.9.2:
|
||||
resolution: {integrity: sha512-JNeLKlrsPxld7oPJCNeF73yHyyYeyP950enWRkTa6WsJ6UohH2NQ1vXZu6lWO9WuA9EMypITyZFZ8KtcTV3y2Q==}
|
||||
engines: {node: '>=14.17.0'}
|
||||
peerDependencies:
|
||||
'@google/generative-ai': ^0.1.1
|
||||
cohere-ai: ^5.0.0 || ^6.0.0 || ^7.0.0
|
||||
openai: ^3.0.0 || ^4.0.0
|
||||
peerDependenciesMeta:
|
||||
'@google/generative-ai':
|
||||
optional: true
|
||||
cohere-ai:
|
||||
optional: true
|
||||
openai:
|
||||
optional: true
|
||||
|
||||
chrome-trace-event@1.0.4:
|
||||
resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==}
|
||||
engines: {node: '>=6.0'}
|
||||
@@ -6568,7 +6553,6 @@ packages:
|
||||
eslint@8.57.0:
|
||||
resolution: {integrity: sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==}
|
||||
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
|
||||
deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options.
|
||||
hasBin: true
|
||||
|
||||
eslint@9.10.0:
|
||||
@@ -18473,17 +18457,7 @@ snapshots:
|
||||
|
||||
chownr@2.0.0: {}
|
||||
|
||||
chromadb@1.8.1(cohere-ai@7.13.1(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)):
|
||||
dependencies:
|
||||
cliui: 8.0.1
|
||||
isomorphic-fetch: 3.0.0(encoding@0.1.13)
|
||||
optionalDependencies:
|
||||
cohere-ai: 7.13.1(encoding@0.1.13)
|
||||
openai: 4.60.1(encoding@0.1.13)(zod@3.23.8)
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
|
||||
chromadb@1.9.2(@google/generative-ai@0.12.0)(cohere-ai@7.13.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.650.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)(zod@3.23.8)):
|
||||
chromadb@1.8.1(@google/generative-ai@0.12.0)(cohere-ai@7.13.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.650.0))(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)(zod@3.23.8)):
|
||||
dependencies:
|
||||
cliui: 8.0.1
|
||||
isomorphic-fetch: 3.0.0(encoding@0.1.13)
|
||||
@@ -18494,6 +18468,16 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
|
||||
chromadb@1.8.1(cohere-ai@7.13.1(encoding@0.1.13))(encoding@0.1.13)(openai@4.60.1(encoding@0.1.13)):
|
||||
dependencies:
|
||||
cliui: 8.0.1
|
||||
isomorphic-fetch: 3.0.0(encoding@0.1.13)
|
||||
optionalDependencies:
|
||||
cohere-ai: 7.13.1(encoding@0.1.13)
|
||||
openai: 4.60.1(encoding@0.1.13)(zod@3.23.8)
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
|
||||
chrome-trace-event@1.0.4: {}
|
||||
|
||||
ci-info@3.9.0: {}
|
||||
@@ -19534,7 +19518,7 @@ snapshots:
|
||||
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
|
||||
eslint: 8.57.0
|
||||
eslint-import-resolver-node: 0.3.9
|
||||
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
|
||||
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
|
||||
eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
|
||||
eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
|
||||
eslint-plugin-react: 7.35.0(eslint@8.57.0)
|
||||
@@ -19582,25 +19566,6 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0):
|
||||
dependencies:
|
||||
'@nolyfill/is-core-module': 1.0.39
|
||||
debug: 4.3.7
|
||||
enhanced-resolve: 5.17.1
|
||||
eslint: 8.57.0
|
||||
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
|
||||
fast-glob: 3.3.2
|
||||
get-tsconfig: 4.8.0
|
||||
is-bun-module: 1.1.0
|
||||
is-glob: 4.0.3
|
||||
optionalDependencies:
|
||||
eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
|
||||
transitivePeerDependencies:
|
||||
- '@typescript-eslint/parser'
|
||||
- eslint-import-resolver-node
|
||||
- eslint-import-resolver-webpack
|
||||
- supports-color
|
||||
|
||||
eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
|
||||
dependencies:
|
||||
'@nolyfill/is-core-module': 1.0.39
|
||||
@@ -19620,17 +19585,6 @@ snapshots:
|
||||
- eslint-import-resolver-webpack
|
||||
- supports-color
|
||||
|
||||
eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
|
||||
dependencies:
|
||||
debug: 3.2.7
|
||||
optionalDependencies:
|
||||
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
|
||||
eslint: 8.57.0
|
||||
eslint-import-resolver-node: 0.3.9
|
||||
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
|
||||
dependencies:
|
||||
debug: 3.2.7
|
||||
@@ -19652,7 +19606,7 @@ snapshots:
|
||||
doctrine: 2.1.0
|
||||
eslint: 8.57.0
|
||||
eslint-import-resolver-node: 0.3.9
|
||||
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
|
||||
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
|
||||
hasown: 2.0.2
|
||||
is-core-module: 2.15.1
|
||||
is-glob: 4.0.3
|
||||
@@ -20418,7 +20372,7 @@ snapshots:
|
||||
defu: 6.1.4
|
||||
node-fetch-native: 1.6.4
|
||||
nypm: 0.3.11
|
||||
ohash: 1.1.4
|
||||
ohash: 1.1.3
|
||||
pathe: 1.1.2
|
||||
tar: 6.2.1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user