mirror of
https://github.com/run-llama/LlamaIndexTS.git
synced 2026-07-13 22:17:48 -04:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Yang
|
20644eabc4 | ||
|
Alex Yang
|
93e86469aa | ||
|
Marcus Schiesser
|
2532e0e01f | ||
|
Marcus Schiesser
|
b3fda249a0 |
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"llamaindex": patch
|
||||
---
|
||||
|
||||
feat: add splitByPage mode to LlamaParseReader
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@llamaindex/core": patch
|
||||
"@llamaindex/core-tests": patch
|
||||
---
|
||||
|
||||
fix: handling errors in splitBySentenceTokenizer
|
||||
+1
-6
@@ -31,12 +31,7 @@ module.exports = {
|
||||
"@typescript-eslint/ban-types": "off",
|
||||
"no-array-constructor": "off",
|
||||
"@typescript-eslint/no-array-constructor": "off",
|
||||
"@typescript-eslint/no-base-to-string": [
|
||||
"error",
|
||||
{
|
||||
ignoredTypeNames: ["Error", "RegExp", "URL", "URLSearchParams"],
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/no-base-to-string": "off",
|
||||
"@typescript-eslint/no-duplicate-enum-values": "off",
|
||||
"@typescript-eslint/no-duplicate-type-constituents": "off",
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
|
||||
@@ -164,7 +164,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
|
||||
|
||||
- [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
|
||||
|
||||
- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
|
||||
- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
|
||||
|
||||
- [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
|
||||
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# docs
|
||||
|
||||
## 0.0.60
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.0.59
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.0.58
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.0.57
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.0.56
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.0.55
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.0.54
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.0.53
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.0.52
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "docs",
|
||||
"version": "0.0.60",
|
||||
"version": "0.0.52",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"docusaurus": "docusaurus",
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# Weaviate Vector Store
|
||||
|
||||
Here are two sample scripts which work with loading and querying data from a Weaviate Vector Store.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- An Weaviate Vector Database
|
||||
- Hosted https://weaviate.io/
|
||||
- Self Hosted https://weaviate.io/developers/weaviate/installation/docker-compose#starter-docker-compose-file
|
||||
- An OpenAI API Key
|
||||
|
||||
## Setup
|
||||
|
||||
1. Set your env variables:
|
||||
|
||||
- `WEAVIATE_CLUSTER_URL`: Address of your Weaviate Vector Store (like localhost:8080)
|
||||
- `WEAVIATE_API_KEY`: Your Weaviate API key
|
||||
- `OPENAI_API_KEY`: Your OpenAI key
|
||||
|
||||
2. `cd` Into the `examples` directory
|
||||
3. run `npm i`
|
||||
|
||||
## Load the data
|
||||
|
||||
This sample loads the same dataset of movie reviews as sample dataset
|
||||
|
||||
run `npx tsx weaviate/load`
|
||||
|
||||
## Use RAG to Query the data
|
||||
|
||||
run `npx tsx weaviate/query`
|
||||
@@ -1,23 +0,0 @@
|
||||
import {
|
||||
PapaCSVReader,
|
||||
storageContextFromDefaults,
|
||||
VectorStoreIndex,
|
||||
WeaviateVectorStore,
|
||||
} from "llamaindex";
|
||||
|
||||
const indexName = "MovieReviews";
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const reader = new PapaCSVReader(false);
|
||||
const docs = await reader.loadData("./data/movie_reviews.csv");
|
||||
const vectorStore = new WeaviateVectorStore({ indexName });
|
||||
const storageContext = await storageContextFromDefaults({ vectorStore });
|
||||
await VectorStoreIndex.fromDocuments(docs, { storageContext });
|
||||
console.log("Successfully loaded data into Weaviate");
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -1,46 +0,0 @@
|
||||
import { VectorStoreIndex, WeaviateVectorStore } from "llamaindex";
|
||||
|
||||
const indexName = "MovieReviews";
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const query = "Get all movie titles.";
|
||||
const vectorStore = new WeaviateVectorStore({ indexName });
|
||||
const index = await VectorStoreIndex.fromVectorStore(vectorStore);
|
||||
const retriever = index.asRetriever({ similarityTopK: 20 });
|
||||
|
||||
const queryEngine = index.asQueryEngine({ retriever });
|
||||
const results = await queryEngine.query({ query });
|
||||
console.log(`Query from ${results.sourceNodes?.length} nodes`);
|
||||
console.log(results.response);
|
||||
|
||||
console.log("\n=====\nQuerying the index with filters");
|
||||
const queryEngineWithFilters = index.asQueryEngine({
|
||||
retriever,
|
||||
preFilters: {
|
||||
filters: [
|
||||
{
|
||||
key: "document_id",
|
||||
value: "./data/movie_reviews.csv_37",
|
||||
operator: "==",
|
||||
},
|
||||
{
|
||||
key: "document_id",
|
||||
value: "./data/movie_reviews.csv_21",
|
||||
operator: "==",
|
||||
},
|
||||
],
|
||||
condition: "or",
|
||||
},
|
||||
});
|
||||
const resultAfterFilter = await queryEngineWithFilters.query({
|
||||
query: "Get all movie titles.",
|
||||
});
|
||||
console.log(`Query from ${resultAfterFilter.sourceNodes?.length} nodes`);
|
||||
console.log(resultAfterFilter.response);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -1,13 +1,5 @@
|
||||
# @llamaindex/autotool
|
||||
|
||||
## 2.0.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 2.0.0
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,71 +1,5 @@
|
||||
# @llamaindex/autotool-02-next-example
|
||||
|
||||
## 0.1.44
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
- @llamaindex/autotool@2.0.1
|
||||
|
||||
## 0.1.43
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
- @llamaindex/autotool@2.0.1
|
||||
|
||||
## 0.1.42
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
- @llamaindex/autotool@2.0.1
|
||||
|
||||
## 0.1.41
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- @llamaindex/autotool@2.0.1
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.1.40
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
- @llamaindex/autotool@2.0.0
|
||||
|
||||
## 0.1.39
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
- @llamaindex/autotool@2.0.0
|
||||
|
||||
## 0.1.38
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
- @llamaindex/autotool@2.0.0
|
||||
|
||||
## 0.1.37
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
- @llamaindex/autotool@2.0.0
|
||||
|
||||
## 0.1.36
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/autotool-02-next-example",
|
||||
"private": true,
|
||||
"version": "0.1.44",
|
||||
"version": "0.1.36",
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/autotool",
|
||||
"type": "module",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.0",
|
||||
"description": "auto transpile your JS function to LLM Agent compatible",
|
||||
"files": [
|
||||
"dist",
|
||||
@@ -51,7 +51,7 @@
|
||||
"unplugin": "^1.10.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"llamaindex": "^0.5.19",
|
||||
"llamaindex": "^0.5.11",
|
||||
"openai": "^4",
|
||||
"typescript": "^4"
|
||||
},
|
||||
@@ -70,7 +70,7 @@
|
||||
"@swc/types": "^0.1.8",
|
||||
"@types/json-schema": "^7.0.15",
|
||||
"@types/node": "^20.12.11",
|
||||
"bunchee": "5.3.1",
|
||||
"bunchee": "5.3.0-beta.0",
|
||||
"llamaindex": "workspace:*",
|
||||
"next": "14.2.5",
|
||||
"rollup": "^4.18.0",
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
# @llamaindex/cloud
|
||||
|
||||
## 0.2.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
|
||||
## 0.2.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/cloud",
|
||||
"version": "0.2.2",
|
||||
"version": "0.2.1",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
@@ -35,6 +35,6 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@hey-api/openapi-ts": "^0.48.0",
|
||||
"bunchee": "5.3.1"
|
||||
"bunchee": "5.3.0-beta.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,35 +1,5 @@
|
||||
# @llamaindex/community
|
||||
|
||||
## 0.0.30
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [e27e7dd]
|
||||
- @llamaindex/core@0.1.9
|
||||
|
||||
## 0.0.29
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
- Updated dependencies [58abc57]
|
||||
- @llamaindex/core@0.1.8
|
||||
- @llamaindex/env@0.1.9
|
||||
|
||||
## 0.0.28
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [04b2f8e]
|
||||
- @llamaindex/core@0.1.7
|
||||
|
||||
## 0.0.27
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [0452af9]
|
||||
- @llamaindex/core@0.1.6
|
||||
|
||||
## 0.0.26
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/community",
|
||||
"description": "Community package for LlamaIndexTS",
|
||||
"version": "0.0.30",
|
||||
"version": "0.0.26",
|
||||
"type": "module",
|
||||
"types": "dist/type/index.d.ts",
|
||||
"main": "dist/cjs/index.js",
|
||||
@@ -43,7 +43,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.2",
|
||||
"bunchee": "5.3.1"
|
||||
"bunchee": "5.3.0-beta.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-bedrock-runtime": "^3.613.0",
|
||||
|
||||
@@ -1,31 +1,5 @@
|
||||
# @llamaindex/core
|
||||
|
||||
## 0.1.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- e27e7dd: chore: bump `natural` to 8.0.1
|
||||
|
||||
## 0.1.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
- Updated dependencies [58abc57]
|
||||
- @llamaindex/env@0.1.9
|
||||
|
||||
## 0.1.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 04b2f8e: Fix issue with metadata included after sentence splitter
|
||||
|
||||
## 0.1.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 0452af9: fix: handling errors in splitBySentenceTokenizer
|
||||
|
||||
## 0.1.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/core",
|
||||
"type": "module",
|
||||
"version": "0.1.9",
|
||||
"version": "0.1.5",
|
||||
"description": "LlamaIndex Core Module",
|
||||
"exports": {
|
||||
"./node-parser": {
|
||||
@@ -131,8 +131,8 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"ajv": "^8.16.0",
|
||||
"bunchee": "5.3.1",
|
||||
"natural": "^8.0.1"
|
||||
"bunchee": "5.3.0-beta.0",
|
||||
"natural": "^7.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@llamaindex/env": "workspace:*",
|
||||
|
||||
@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
|
||||
return nodes.reduce<TextNode[]>((allNodes, node) => {
|
||||
const metadataStr = this.getMetadataString(node);
|
||||
const splits = this.splitTextMetadataAware(
|
||||
node.getContent(MetadataMode.NONE),
|
||||
node.getContent(MetadataMode.ALL),
|
||||
metadataStr,
|
||||
);
|
||||
return allNodes.concat(buildNodeFromSplits(splits, node));
|
||||
|
||||
-1
@@ -1,5 +1,4 @@
|
||||
declare class SentenceTokenizer {
|
||||
constructor(abbreviations?: string[]);
|
||||
tokenize(text: string): string[];
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,222 +0,0 @@
|
||||
var __getOwnPropNames = Object.getOwnPropertyNames;
|
||||
var __commonJS = (cb, mod) =>
|
||||
function __require() {
|
||||
return (
|
||||
mod ||
|
||||
(0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod),
|
||||
mod.exports
|
||||
);
|
||||
};
|
||||
|
||||
// lib/natural/tokenizers/tokenizer.js
|
||||
var require_tokenizer = __commonJS({
|
||||
"lib/natural/tokenizers/tokenizer.js"(exports, module) {
|
||||
"use strict";
|
||||
var Tokenizer = class {
|
||||
trim(array) {
|
||||
while (array[array.length - 1] === "") {
|
||||
array.pop();
|
||||
}
|
||||
while (array[0] === "") {
|
||||
array.shift();
|
||||
}
|
||||
return array;
|
||||
}
|
||||
};
|
||||
module.exports = Tokenizer;
|
||||
},
|
||||
});
|
||||
|
||||
// lib/natural/tokenizers/sentence_tokenizer.js
|
||||
var require_sentence_tokenizer = __commonJS({
|
||||
"lib/natural/tokenizers/sentence_tokenizer.js"(exports, module) {
|
||||
var Tokenizer = require_tokenizer();
|
||||
var NUM = "NUMBER";
|
||||
var DELIM = "DELIM";
|
||||
var URI = "URI";
|
||||
var ABBREV = "ABBREV";
|
||||
var DEBUG = false;
|
||||
function generateUniqueCode(base, index) {
|
||||
return `{{${base}_${index}}}`;
|
||||
}
|
||||
function escapeRegExp(string) {
|
||||
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
var SentenceTokenizer = class extends Tokenizer {
|
||||
constructor(abbreviations) {
|
||||
super();
|
||||
if (abbreviations) {
|
||||
this.abbreviations = abbreviations;
|
||||
} else {
|
||||
this.abbreviations = [];
|
||||
}
|
||||
this.replacementMap = null;
|
||||
this.replacementCounter = 0;
|
||||
}
|
||||
replaceUrisWithPlaceholders(text) {
|
||||
const urlPattern =
|
||||
/(https?:\/\/\S+|www\.\S+|ftp:\/\/\S+|(mailto:)?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|file:\/\/\S+)/gi;
|
||||
const modifiedText = text.replace(urlPattern, (match) => {
|
||||
const placeholder = generateUniqueCode(
|
||||
URI,
|
||||
this.replacementCounter++,
|
||||
);
|
||||
this.replacementMap.set(placeholder, match);
|
||||
return placeholder;
|
||||
});
|
||||
return modifiedText;
|
||||
}
|
||||
replaceAbbreviations(text) {
|
||||
if (this.abbreviations.length === 0) {
|
||||
return text;
|
||||
}
|
||||
const pattern = new RegExp(
|
||||
`(${this.abbreviations.map((abbrev) => escapeRegExp(abbrev)).join("|")})`,
|
||||
"gi",
|
||||
);
|
||||
const replacedText = text.replace(pattern, (match) => {
|
||||
const code = generateUniqueCode(ABBREV, this.replacementCounter++);
|
||||
this.replacementMap.set(code, match);
|
||||
return code;
|
||||
});
|
||||
return replacedText;
|
||||
}
|
||||
replaceDelimitersWithPlaceholders(text) {
|
||||
const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g;
|
||||
const modifiedText = text.replace(
|
||||
delimiterPattern,
|
||||
(match, p1, p2, p3) => {
|
||||
const placeholder = generateUniqueCode(
|
||||
DELIM,
|
||||
this.replacementCounter++,
|
||||
);
|
||||
this.delimiterMap.set(placeholder, p1 + p2 + p3);
|
||||
return placeholder;
|
||||
},
|
||||
);
|
||||
return modifiedText;
|
||||
}
|
||||
splitOnPlaceholders(text, placeholders) {
|
||||
if (this.delimiterMap.size === 0) {
|
||||
return [text];
|
||||
}
|
||||
const keys = Array.from(this.delimiterMap.keys());
|
||||
const pattern = new RegExp(`(${keys.map(escapeRegExp).join("|")})`);
|
||||
const parts = text.split(pattern);
|
||||
const sentences = [];
|
||||
for (let i = 0; i < parts.length; i += 2) {
|
||||
const sentence = parts[i];
|
||||
const placeholder = parts[i + 1] || "";
|
||||
sentences.push(sentence + placeholder);
|
||||
}
|
||||
return sentences;
|
||||
}
|
||||
replaceNumbersWithCode(text) {
|
||||
const numberPattern = /\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b/g;
|
||||
const replacedText = text.replace(numberPattern, (match) => {
|
||||
const code = generateUniqueCode(NUM, this.replacementCounter++);
|
||||
this.replacementMap.set(code, match);
|
||||
return code;
|
||||
});
|
||||
return replacedText;
|
||||
}
|
||||
revertReplacements(text) {
|
||||
let originalText = text;
|
||||
for (const [
|
||||
placeholder,
|
||||
replacement,
|
||||
] of this.replacementMap.entries()) {
|
||||
const pattern = new RegExp(escapeRegExp(placeholder), "g");
|
||||
originalText = originalText.replace(pattern, replacement);
|
||||
}
|
||||
return originalText;
|
||||
}
|
||||
revertDelimiters(text) {
|
||||
let originalText = text;
|
||||
for (const [placeholder, replacement] of this.delimiterMap.entries()) {
|
||||
const pattern = new RegExp(escapeRegExp(placeholder), "g");
|
||||
originalText = originalText.replace(pattern, replacement);
|
||||
}
|
||||
return originalText;
|
||||
}
|
||||
tokenize(text) {
|
||||
this.replacementCounter = 0;
|
||||
this.replacementMap = /* @__PURE__ */ new Map();
|
||||
this.delimiterMap = /* @__PURE__ */ new Map();
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"---Start of sentence tokenization-----------------------",
|
||||
);
|
||||
DEBUG && console.log("Original input: >>>" + text + "<<<");
|
||||
const result1 = this.replaceAbbreviations(text);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 1: replacing abbreviations: " + JSON.stringify(result1),
|
||||
);
|
||||
const result2 = this.replaceUrisWithPlaceholders(result1);
|
||||
DEBUG &&
|
||||
console.log("Phase 2: replacing URIs: " + JSON.stringify(result2));
|
||||
const result3 = this.replaceNumbersWithCode(result2);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 3: replacing numbers with placeholders: " +
|
||||
JSON.stringify(result3),
|
||||
);
|
||||
const result4 = this.replaceDelimitersWithPlaceholders(result3);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 4: replacing delimiters with placeholders: " +
|
||||
JSON.stringify(result4),
|
||||
);
|
||||
const sentences = this.splitOnPlaceholders(result4);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 5: splitting into sentences on placeholders: " +
|
||||
JSON.stringify(sentences),
|
||||
);
|
||||
const newSentences = sentences.map((s) => {
|
||||
const s1 = this.revertReplacements(s);
|
||||
return this.revertDelimiters(s1);
|
||||
});
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 6: replacing back abbreviations, URIs, numbers and delimiters: " +
|
||||
JSON.stringify(newSentences),
|
||||
);
|
||||
const trimmedSentences = this.trim(newSentences);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 7: trimming array of empty sentences: " +
|
||||
JSON.stringify(trimmedSentences),
|
||||
);
|
||||
const trimmedSentences2 = trimmedSentences.map((sent) => sent.trim());
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"Phase 8: trimming sentences from surrounding whitespace: " +
|
||||
JSON.stringify(trimmedSentences2),
|
||||
);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"---End of sentence tokenization--------------------------",
|
||||
);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"---Replacement map---------------------------------------",
|
||||
);
|
||||
DEBUG && console.log([...this.replacementMap.entries()]);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"---Delimiter map-----------------------------------------",
|
||||
);
|
||||
DEBUG && console.log([...this.delimiterMap.entries()]);
|
||||
DEBUG &&
|
||||
console.log(
|
||||
"---------------------------------------------------------",
|
||||
);
|
||||
return trimmedSentences2;
|
||||
}
|
||||
};
|
||||
module.exports = SentenceTokenizer;
|
||||
},
|
||||
});
|
||||
export default require_sentence_tokenizer();
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { TextSplitter } from "./base";
|
||||
import SentenceTokenizer from "./sentence_tokenizer";
|
||||
import SentenceTokenizerNew from "./sentence-tokenizer-parser.js";
|
||||
|
||||
export type TextSplitterFn = (text: string) => string[];
|
||||
|
||||
@@ -31,17 +31,11 @@ export const splitByChar = (): TextSplitterFn => {
|
||||
return (text: string) => text.split("");
|
||||
};
|
||||
|
||||
let sentenceTokenizer: SentenceTokenizer | null = null;
|
||||
let sentenceTokenizer: SentenceTokenizerNew | null = null;
|
||||
|
||||
export const splitBySentenceTokenizer = (): TextSplitterFn => {
|
||||
if (!sentenceTokenizer) {
|
||||
sentenceTokenizer = new SentenceTokenizer([
|
||||
"i.e.",
|
||||
"etc.",
|
||||
"vs.",
|
||||
"Inc.",
|
||||
"A.S.A.P.",
|
||||
]);
|
||||
sentenceTokenizer = new SentenceTokenizerNew();
|
||||
}
|
||||
const tokenizer = sentenceTokenizer;
|
||||
return (text: string) => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export * from "./node";
|
||||
export { FileReader, TransformComponent, type BaseReader } from "./type";
|
||||
export { TransformComponent } from "./type";
|
||||
export { EngineResponse } from "./type/engine–response";
|
||||
export * from "./zod";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { fs, path, randomUUID } from "@llamaindex/env";
|
||||
import type { BaseNode, Document } from "./node";
|
||||
import { randomUUID } from "@llamaindex/env";
|
||||
import type { BaseNode } from "./node";
|
||||
|
||||
interface TransformComponentSignature {
|
||||
<Options extends Record<string, unknown>>(
|
||||
@@ -28,37 +28,3 @@ export class TransformComponent {
|
||||
return transform;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A reader takes imports data into Document objects.
|
||||
*/
|
||||
export interface BaseReader {
|
||||
loadData(...args: unknown[]): Promise<Document[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A FileReader takes file paths and imports data into Document objects.
|
||||
*/
|
||||
export abstract class FileReader implements BaseReader {
|
||||
abstract loadDataAsContent(
|
||||
fileContent: Uint8Array,
|
||||
fileName?: string,
|
||||
): Promise<Document[]>;
|
||||
|
||||
async loadData(filePath: string): Promise<Document[]> {
|
||||
const fileContent = await fs.readFile(filePath);
|
||||
const fileName = path.basename(filePath);
|
||||
const docs = await this.loadDataAsContent(fileContent, fileName);
|
||||
docs.forEach(FileReader.addMetaData(filePath));
|
||||
return docs;
|
||||
}
|
||||
|
||||
static addMetaData(filePath: string) {
|
||||
return (doc: Document, index: number) => {
|
||||
// generate id as loadDataAsContent is only responsible for the content
|
||||
doc.id_ = `${filePath}_${index + 1}`;
|
||||
doc.metadata["file_path"] = path.resolve(filePath);
|
||||
doc.metadata["file_name"] = path.basename(filePath);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
-6
@@ -1,11 +1,5 @@
|
||||
# @llamaindex/env
|
||||
|
||||
## 0.1.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
|
||||
## 0.1.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
Vendored
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/env",
|
||||
"description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
|
||||
"version": "0.1.9",
|
||||
"version": "0.1.8",
|
||||
"type": "module",
|
||||
"types": "dist/type/index.d.ts",
|
||||
"main": "dist/cjs/index.js",
|
||||
|
||||
@@ -1,64 +1,5 @@
|
||||
# @llamaindex/experimental
|
||||
|
||||
## 0.0.69
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.0.68
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.0.67
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.0.66
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.0.65
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.0.64
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.0.63
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.0.62
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.0.61
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/experimental",
|
||||
"description": "Experimental package for LlamaIndexTS",
|
||||
"version": "0.0.69",
|
||||
"version": "0.0.61",
|
||||
"type": "module",
|
||||
"types": "dist/type/index.d.ts",
|
||||
"main": "dist/cjs/index.js",
|
||||
|
||||
@@ -162,18 +162,18 @@ export class JSONQueryEngine implements QueryEngine {
|
||||
|
||||
const schema = this.getSchemaContext();
|
||||
|
||||
const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
|
||||
const jsonPathResponseStr = await this.serviceContext.llm.complete({
|
||||
prompt: this.jsonPathPrompt({ query, schema }),
|
||||
});
|
||||
|
||||
if (this.verbose) {
|
||||
console.log(
|
||||
`> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
|
||||
`> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const jsonPathOutput = await this.outputProcessor({
|
||||
llmOutput: jsonPathResponse,
|
||||
llmOutput: jsonPathResponseStr.text,
|
||||
jsonValue: this.jsonValue,
|
||||
});
|
||||
|
||||
@@ -188,7 +188,7 @@ export class JSONQueryEngine implements QueryEngine {
|
||||
prompt: this.responseSynthesisPrompt({
|
||||
query,
|
||||
jsonSchema: schema,
|
||||
jsonPath: jsonPathResponse,
|
||||
jsonPath: jsonPathResponseStr.text,
|
||||
jsonPathValue: JSON.stringify(jsonPathOutput),
|
||||
}),
|
||||
});
|
||||
@@ -199,7 +199,7 @@ export class JSONQueryEngine implements QueryEngine {
|
||||
}
|
||||
|
||||
const responseMetadata = {
|
||||
jsonPathResponse,
|
||||
jsonPathResponseStr,
|
||||
};
|
||||
|
||||
const response = EngineResponse.fromResponse(responseStr, false);
|
||||
|
||||
@@ -1,65 +1,5 @@
|
||||
# llamaindex
|
||||
|
||||
## 0.5.19
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fcbf183: implement llamacloud file service
|
||||
|
||||
## 0.5.18
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 8b66cf4: feat: support organization id in llamacloud index
|
||||
- Updated dependencies [e27e7dd]
|
||||
- @llamaindex/core@0.1.9
|
||||
|
||||
## 0.5.17
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- c654398: Implement Weaviate Vector Store in TS
|
||||
|
||||
## 0.5.16
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 58abc57: fix: align version
|
||||
- Updated dependencies [58abc57]
|
||||
- @llamaindex/cloud@0.2.2
|
||||
- @llamaindex/core@0.1.8
|
||||
- @llamaindex/env@0.1.9
|
||||
|
||||
## 0.5.15
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 01c184c: Add is_empty operator for filtering vector store
|
||||
- 07a275f: chore: bump openai
|
||||
|
||||
## 0.5.14
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
|
||||
|
||||
## 0.5.13
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [04b2f8e]
|
||||
- @llamaindex/core@0.1.7
|
||||
|
||||
## 0.5.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 345300f: feat: add splitByPage mode to LlamaParseReader
|
||||
- da5cfc4: Add metadatafilter options to retriever constructors
|
||||
- da5cfc4: Fix system prompt not used in ContextChatEngine
|
||||
- Updated dependencies [0452af9]
|
||||
- @llamaindex/core@0.1.6
|
||||
|
||||
## 0.5.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# @llamaindex/cloudflare-worker-agent-test
|
||||
|
||||
## 0.0.53
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.0.52
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.0.51
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.0.50
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.0.49
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.0.48
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.0.47
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.0.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.0.45
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/cloudflare-worker-agent-test",
|
||||
"version": "0.0.53",
|
||||
"version": "0.0.45",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# @llamaindex/next-agent-test
|
||||
|
||||
## 0.1.53
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.1.52
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.1.51
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.1.50
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.1.49
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.1.48
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.1.47
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.1.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.1.45
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/next-agent-test",
|
||||
"version": "0.1.53",
|
||||
"version": "0.1.45",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# test-edge-runtime
|
||||
|
||||
## 0.1.52
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.1.51
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.1.50
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.1.49
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.1.48
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.1.47
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.1.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.1.45
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.1.44
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/nextjs-edge-runtime-test",
|
||||
"version": "0.1.52",
|
||||
"version": "0.1.44",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# @llamaindex/next-node-runtime
|
||||
|
||||
## 0.0.34
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.0.33
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.0.32
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.0.31
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.0.30
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.0.29
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.0.28
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.0.27
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.0.26
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/next-node-runtime-test",
|
||||
"version": "0.0.34",
|
||||
"version": "0.0.26",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
|
||||
@@ -1,63 +1,5 @@
|
||||
# @llamaindex/waku-query-engine-test
|
||||
|
||||
## 0.0.53
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [fcbf183]
|
||||
- llamaindex@0.5.19
|
||||
|
||||
## 0.0.52
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [8b66cf4]
|
||||
- llamaindex@0.5.18
|
||||
|
||||
## 0.0.51
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c654398]
|
||||
- llamaindex@0.5.17
|
||||
|
||||
## 0.0.50
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [58abc57]
|
||||
- llamaindex@0.5.16
|
||||
|
||||
## 0.0.49
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [01c184c]
|
||||
- Updated dependencies [07a275f]
|
||||
- llamaindex@0.5.15
|
||||
|
||||
## 0.0.48
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [c825a2f]
|
||||
- llamaindex@0.5.14
|
||||
|
||||
## 0.0.47
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- llamaindex@0.5.13
|
||||
|
||||
## 0.0.46
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [345300f]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- Updated dependencies [da5cfc4]
|
||||
- llamaindex@0.5.12
|
||||
|
||||
## 0.0.45
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@llamaindex/waku-query-engine-test",
|
||||
"version": "0.0.53",
|
||||
"version": "0.0.45",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llamaindex",
|
||||
"version": "0.5.19",
|
||||
"version": "0.5.11",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"keywords": [
|
||||
@@ -55,7 +55,7 @@
|
||||
"md-utils-ts": "^2.0.0",
|
||||
"mongodb": "^6.7.0",
|
||||
"notion-md-crawler": "^1.0.0",
|
||||
"openai": "^4.55.3",
|
||||
"openai": "^4.52.5",
|
||||
"papaparse": "^5.4.1",
|
||||
"pathe": "^1.1.2",
|
||||
"pg": "^8.12.0",
|
||||
@@ -65,7 +65,6 @@
|
||||
"string-strip-html": "^13.4.8",
|
||||
"tiktoken": "^1.0.15",
|
||||
"unpdf": "^0.11.0",
|
||||
"weaviate-client": "^3.1.4",
|
||||
"wikipedia": "^2.1.2",
|
||||
"wink-nlp": "^2.3.0",
|
||||
"zod": "^3.23.8"
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
import {
|
||||
FilesService,
|
||||
PipelinesService,
|
||||
ProjectsService,
|
||||
} from "@llamaindex/cloud/api";
|
||||
import { initService } from "./utils.js";
|
||||
|
||||
export class LLamaCloudFileService {
|
||||
/**
|
||||
* Get list of projects, each project contains a list of pipelines
|
||||
*/
|
||||
public static async getAllProjectsWithPipelines() {
|
||||
initService();
|
||||
try {
|
||||
const projects = await ProjectsService.listProjectsApiV1ProjectsGet();
|
||||
const pipelines =
|
||||
await PipelinesService.searchPipelinesApiV1PipelinesGet();
|
||||
return projects.map((project) => ({
|
||||
...project,
|
||||
pipelines: pipelines.filter((p) => p.project_id === project.id),
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error("Error listing projects and pipelines:", error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to a pipeline in LlamaCloud
|
||||
*/
|
||||
public static async addFileToPipeline(
|
||||
projectId: string,
|
||||
pipelineId: string,
|
||||
uploadFile: File | Blob,
|
||||
customMetadata: Record<string, any> = {},
|
||||
) {
|
||||
initService();
|
||||
const file = await FilesService.uploadFileApiV1FilesPost({
|
||||
projectId,
|
||||
formData: {
|
||||
upload_file: uploadFile,
|
||||
},
|
||||
});
|
||||
const files = [
|
||||
{
|
||||
file_id: file.id,
|
||||
custom_metadata: { file_id: file.id, ...customMetadata },
|
||||
},
|
||||
];
|
||||
await PipelinesService.addFilesToPipelineApiV1PipelinesPipelineIdFilesPut({
|
||||
pipelineId,
|
||||
requestBody: files,
|
||||
});
|
||||
|
||||
// Wait 2s for the file to be processed
|
||||
const maxAttempts = 20;
|
||||
let attempt = 0;
|
||||
while (attempt < maxAttempts) {
|
||||
const result =
|
||||
await PipelinesService.getPipelineFileStatusApiV1PipelinesPipelineIdFilesFileIdStatusGet(
|
||||
{
|
||||
pipelineId,
|
||||
fileId: file.id,
|
||||
},
|
||||
);
|
||||
if (result.status === "ERROR") {
|
||||
throw new Error(`File processing failed: ${JSON.stringify(result)}`);
|
||||
}
|
||||
if (result.status === "SUCCESS") {
|
||||
// File is ingested - return the file id
|
||||
return file.id;
|
||||
}
|
||||
attempt += 1;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100)); // Sleep for 100ms
|
||||
}
|
||||
throw new Error(
|
||||
`File processing did not complete after ${maxAttempts} attempts.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get download URL for a file in LlamaCloud
|
||||
*/
|
||||
public static async getFileUrl(pipelineId: string, filename: string) {
|
||||
initService();
|
||||
const allPipelineFiles =
|
||||
await PipelinesService.listPipelineFilesApiV1PipelinesPipelineIdFilesGet({
|
||||
pipelineId,
|
||||
});
|
||||
const file = allPipelineFiles.find((file) => file.name === filename);
|
||||
if (!file?.file_id) return null;
|
||||
const fileContent =
|
||||
await FilesService.readFileContentApiV1FilesIdContentGet({
|
||||
id: file.file_id,
|
||||
projectId: file.project_id,
|
||||
});
|
||||
return fileContent.url;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
|
||||
import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
|
||||
import { getPipelineCreate } from "./config.js";
|
||||
import type { CloudConstructorParams } from "./constants.js";
|
||||
import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
|
||||
import { getAppBaseUrl, initService } from "./utils.js";
|
||||
|
||||
import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
|
||||
import { SentenceSplitter } from "@llamaindex/core/node-parser";
|
||||
@@ -132,28 +132,18 @@ export class LlamaCloudIndex {
|
||||
await this.waitForPipelineIngestion(verbose, raiseOnError);
|
||||
}
|
||||
|
||||
public async getPipelineId(
|
||||
name?: string,
|
||||
projectName?: string,
|
||||
private async getPipelineId(
|
||||
name: string,
|
||||
projectName: string,
|
||||
): Promise<string> {
|
||||
const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
projectId: await this.getProjectId(projectName),
|
||||
pipelineName: name ?? this.params.name,
|
||||
projectName,
|
||||
pipelineName: name,
|
||||
});
|
||||
|
||||
return pipelines[0].id;
|
||||
}
|
||||
|
||||
public async getProjectId(
|
||||
projectName?: string,
|
||||
organizationId?: string,
|
||||
): Promise<string> {
|
||||
return await getProjectId(
|
||||
projectName ?? this.params.projectName,
|
||||
organizationId ?? this.params.organizationId,
|
||||
);
|
||||
}
|
||||
|
||||
static async fromDocuments(
|
||||
params: {
|
||||
documents: Document[];
|
||||
@@ -178,7 +168,6 @@ export class LlamaCloudIndex {
|
||||
});
|
||||
|
||||
const project = await ProjectsService.upsertProjectApiV1ProjectsPut({
|
||||
organizationId: params.organizationId,
|
||||
requestBody: {
|
||||
name: params.projectName ?? "default",
|
||||
},
|
||||
|
||||
@@ -11,17 +11,16 @@ import { extractText, wrapEventCaller } from "@llamaindex/core/utils";
|
||||
import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
|
||||
import type { ClientParams, CloudConstructorParams } from "./constants.js";
|
||||
import { DEFAULT_PROJECT_NAME } from "./constants.js";
|
||||
import { getProjectId, initService } from "./utils.js";
|
||||
import { initService } from "./utils.js";
|
||||
|
||||
export type CloudRetrieveParams = Omit<
|
||||
RetrievalParams,
|
||||
"query" | "search_filters" | "dense_similarity_top_k"
|
||||
> & { similarityTopK?: number; filters?: MetadataFilters };
|
||||
"query" | "searchFilters" | "className" | "denseSimilarityTopK"
|
||||
> & { similarityTopK?: number };
|
||||
|
||||
export class LlamaCloudRetriever implements BaseRetriever {
|
||||
clientParams: ClientParams;
|
||||
retrieveParams: CloudRetrieveParams;
|
||||
organizationId?: string;
|
||||
projectName: string = DEFAULT_PROJECT_NAME;
|
||||
pipelineName: string;
|
||||
|
||||
@@ -50,9 +49,6 @@ export class LlamaCloudRetriever implements BaseRetriever {
|
||||
if (params.projectName) {
|
||||
this.projectName = params.projectName;
|
||||
}
|
||||
if (params.organizationId) {
|
||||
this.organizationId = params.organizationId;
|
||||
}
|
||||
}
|
||||
|
||||
@wrapEventCaller
|
||||
@@ -61,7 +57,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
|
||||
preFilters,
|
||||
}: RetrieveParams): Promise<NodeWithScore[]> {
|
||||
const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
|
||||
projectId: await getProjectId(this.projectName, this.organizationId),
|
||||
projectName: this.projectName,
|
||||
pipelineName: this.pipelineName,
|
||||
});
|
||||
|
||||
@@ -88,9 +84,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
|
||||
requestBody: {
|
||||
...this.retrieveParams,
|
||||
query: extractText(query),
|
||||
search_filters:
|
||||
this.retrieveParams.filters ?? (preFilters as MetadataFilters),
|
||||
dense_similarity_top_k: this.retrieveParams.similarityTopK,
|
||||
search_filters: preFilters as MetadataFilters,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -8,6 +8,5 @@ export type ClientParams = { apiKey?: string; baseUrl?: string };
|
||||
export type CloudConstructorParams = {
|
||||
name: string;
|
||||
projectName: string;
|
||||
organizationId?: string;
|
||||
serviceContext?: ServiceContext;
|
||||
} & ClientParams;
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
export type { CloudConstructorParams } from "./constants.js";
|
||||
export { LLamaCloudFileService } from "./LLamaCloudFileService.js";
|
||||
export { LlamaCloudIndex } from "./LlamaCloudIndex.js";
|
||||
export {
|
||||
LlamaCloudRetriever,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { OpenAPI, ProjectsService } from "@llamaindex/cloud/api";
|
||||
import { OpenAPI } from "@llamaindex/cloud/api";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ClientParams } from "./constants.js";
|
||||
import { DEFAULT_BASE_URL } from "./constants.js";
|
||||
@@ -20,31 +20,3 @@ export function initService({ apiKey, baseUrl }: ClientParams = {}) {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getProjectId(
|
||||
projectName: string,
|
||||
organizationId?: string,
|
||||
): Promise<string> {
|
||||
const projects = await ProjectsService.listProjectsApiV1ProjectsGet({
|
||||
projectName: projectName,
|
||||
organizationId: organizationId,
|
||||
});
|
||||
|
||||
if (projects.length === 0) {
|
||||
throw new Error(
|
||||
`Unknown project name ${projectName}. Please confirm a managed project with this name exists.`,
|
||||
);
|
||||
} else if (projects.length > 1) {
|
||||
throw new Error(
|
||||
`Multiple projects found with name ${projectName}. Please specify organization_id.`,
|
||||
);
|
||||
}
|
||||
|
||||
const project = projects[0];
|
||||
|
||||
if (!project.id) {
|
||||
throw new Error(`No project found with name ${projectName}`);
|
||||
}
|
||||
|
||||
return project.id;
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
|
||||
private async getImageInput(
|
||||
image: ImageType,
|
||||
): Promise<{ bytes: string } | { url: string }> {
|
||||
if (isLocal(image) || image instanceof Blob) {
|
||||
if (isLocal(image)) {
|
||||
const base64 = await imageToDataUrl(image);
|
||||
const bytes = base64.split(",")[1];
|
||||
return { bytes };
|
||||
|
||||
@@ -126,7 +126,7 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
|
||||
if (!this.systemPrompt) return message;
|
||||
return {
|
||||
...message,
|
||||
content: this.systemPrompt.trim() + "\n" + extractText(message.content),
|
||||
content: this.systemPrompt.trim() + "\n" + message.content,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
|
||||
const responses: EngineResponse[] = [];
|
||||
for (let i = 0; i < result.selections.length; i++) {
|
||||
const engineInd = result.selections[i];
|
||||
const logStr = `Selecting query engine ${engineInd.index}: ${result.selections[i].index}.`;
|
||||
const logStr = `Selecting query engine ${engineInd}: ${result.selections[i]}.`;
|
||||
|
||||
if (this.verbose) {
|
||||
console.log(logStr + "\n");
|
||||
|
||||
@@ -119,15 +119,15 @@ export class SubQuestionQueryEngine
|
||||
return null;
|
||||
}
|
||||
|
||||
const responseValue = await queryEngine?.call?.({
|
||||
const responseText = await queryEngine?.call?.({
|
||||
query: question,
|
||||
});
|
||||
|
||||
if (responseValue == null) {
|
||||
if (!responseText) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const nodeText = `Sub question: ${question}\nResponse: ${typeof responseValue === "string" ? responseValue : JSON.stringify(responseValue)}`;
|
||||
const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
|
||||
const node = new TextNode({ text: nodeText });
|
||||
return { node, score: 0 };
|
||||
} catch (error) {
|
||||
|
||||
@@ -78,7 +78,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
|
||||
serviceContext: this.serviceContext,
|
||||
});
|
||||
|
||||
const queryResponse = `Question: ${extractText(query)}\nResponse: ${response}`;
|
||||
const queryResponse = `Question: ${query}\nResponse: ${response}`;
|
||||
|
||||
const queryEngine = index.asQueryEngine();
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import type { AgentEndEvent, AgentStartEvent } from "./agent/types.js";
|
||||
import type { RetrievalEndEvent, RetrievalStartEvent } from "./llm/types.js";
|
||||
|
||||
export * from "@llamaindex/core/schema";
|
||||
|
||||
declare module "@llamaindex/core/global" {
|
||||
export interface LlamaIndexEventMaps {
|
||||
"retrieve-start": RetrievalStartEvent;
|
||||
|
||||
@@ -386,7 +386,6 @@ export type VectorIndexRetrieverOptions = {
|
||||
index: VectorStoreIndex;
|
||||
similarityTopK?: number;
|
||||
topK?: TopKMap;
|
||||
filters?: MetadataFilters;
|
||||
};
|
||||
|
||||
export class VectorIndexRetriever implements BaseRetriever {
|
||||
@@ -394,21 +393,14 @@ export class VectorIndexRetriever implements BaseRetriever {
|
||||
topK: TopKMap;
|
||||
|
||||
serviceContext?: ServiceContext;
|
||||
filters?: MetadataFilters;
|
||||
|
||||
constructor({
|
||||
index,
|
||||
similarityTopK,
|
||||
topK,
|
||||
filters,
|
||||
}: VectorIndexRetrieverOptions) {
|
||||
constructor({ index, similarityTopK, topK }: VectorIndexRetrieverOptions) {
|
||||
this.index = index;
|
||||
this.serviceContext = this.index.serviceContext;
|
||||
this.topK = topK ?? {
|
||||
[ModalityType.TEXT]: similarityTopK ?? DEFAULT_SIMILARITY_TOP_K,
|
||||
[ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
|
||||
};
|
||||
this.filters = filters;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -451,7 +443,7 @@ export class VectorIndexRetriever implements BaseRetriever {
|
||||
query: MessageContent,
|
||||
type: ModalityType,
|
||||
vectorStore: VectorStore,
|
||||
filters?: MetadataFilters,
|
||||
preFilters?: MetadataFilters,
|
||||
): Promise<NodeWithScore[]> {
|
||||
// convert string message to multi-modal format
|
||||
if (typeof query === "string") {
|
||||
@@ -468,7 +460,7 @@ export class VectorIndexRetriever implements BaseRetriever {
|
||||
queryEmbedding,
|
||||
mode: VectorStoreQueryMode.DEFAULT,
|
||||
similarityTopK: this.topK[type],
|
||||
filters: this.filters ?? filters ?? undefined,
|
||||
filters: preFilters ?? undefined,
|
||||
});
|
||||
nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { BaseReader, TransformComponent } from "@llamaindex/core/schema";
|
||||
import type { TransformComponent } from "@llamaindex/core/schema";
|
||||
import {
|
||||
ModalityType,
|
||||
splitNodesByType,
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type Document,
|
||||
type Metadata,
|
||||
} from "@llamaindex/core/schema";
|
||||
import type { BaseReader } from "../readers/type.js";
|
||||
import type { BaseDocumentStore } from "../storage/docStore/types.js";
|
||||
import type {
|
||||
VectorStore,
|
||||
@@ -106,7 +107,6 @@ export class IngestionPipeline {
|
||||
inputNodes.push(this.documents);
|
||||
}
|
||||
if (this.reader) {
|
||||
// fixme: empty parameter might cause error
|
||||
inputNodes.push(await this.reader.loadData());
|
||||
}
|
||||
return inputNodes.flat();
|
||||
|
||||
@@ -18,7 +18,6 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
|
||||
openAIModel: "gpt-3.5-turbo-16k",
|
||||
},
|
||||
"gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
|
||||
"gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
|
||||
"gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
|
||||
"gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
|
||||
"gpt-4-turbo": {
|
||||
@@ -41,10 +40,6 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
|
||||
contextWindow: 128000,
|
||||
openAIModel: "gpt-4o-2024-05-13",
|
||||
},
|
||||
"gpt-4o-mini-2024-07-18": {
|
||||
contextWindow: 128000,
|
||||
openAIModel: "gpt-4o-mini-2024-07-18",
|
||||
},
|
||||
};
|
||||
|
||||
const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
|
||||
@@ -78,7 +73,6 @@ const ALL_AZURE_API_VERSIONS = [
|
||||
"2024-03-01-preview",
|
||||
"2024-04-01-preview",
|
||||
"2024-05-01-preview",
|
||||
"2024-06-01",
|
||||
];
|
||||
|
||||
const DEFAULT_API_VERSION = "2023-05-15";
|
||||
|
||||
@@ -6,7 +6,6 @@ import type {
|
||||
ClientOptions as OpenAIClientOptions,
|
||||
} from "openai";
|
||||
import { AzureOpenAI, OpenAI as OrigOpenAI } from "openai";
|
||||
import type { ChatModel } from "openai/resources/chat/chat";
|
||||
|
||||
import {
|
||||
type BaseTool,
|
||||
@@ -109,24 +108,16 @@ export const GPT4_MODELS = {
|
||||
"gpt-4o-2024-05-13": { contextWindow: 128000 },
|
||||
"gpt-4o-mini": { contextWindow: 128000 },
|
||||
"gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
|
||||
"gpt-4o-2024-08-06": { contextWindow: 128000 },
|
||||
"gpt-4o-2024-09-14": { contextWindow: 128000 },
|
||||
"gpt-4o-2024-10-14": { contextWindow: 128000 },
|
||||
"gpt-4-0613": { contextWindow: 128000 },
|
||||
"gpt-4-turbo-2024-04-09": { contextWindow: 128000 },
|
||||
"gpt-4-0314": { contextWindow: 128000 },
|
||||
"gpt-4-32k-0314": { contextWindow: 32768 },
|
||||
};
|
||||
|
||||
// NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future
|
||||
export const GPT35_MODELS = {
|
||||
"gpt-3.5-turbo": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo": { contextWindow: 4096 },
|
||||
"gpt-3.5-turbo-0613": { contextWindow: 4096 },
|
||||
"gpt-3.5-turbo-16k": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo-16k-0613": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo-1106": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo-0125": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo-0301": { contextWindow: 16385 },
|
||||
"gpt-3.5-turbo-16k": { contextWindow: 16384 },
|
||||
"gpt-3.5-turbo-16k-0613": { contextWindow: 16384 },
|
||||
"gpt-3.5-turbo-1106": { contextWindow: 16384 },
|
||||
"gpt-3.5-turbo-0125": { contextWindow: 16384 },
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -135,7 +126,7 @@ export const GPT35_MODELS = {
|
||||
export const ALL_AVAILABLE_OPENAI_MODELS = {
|
||||
...GPT4_MODELS,
|
||||
...GPT35_MODELS,
|
||||
} satisfies Record<ChatModel, { contextWindow: number }>;
|
||||
};
|
||||
|
||||
export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
|
||||
let model: string;
|
||||
@@ -166,10 +157,8 @@ export type OpenAIAdditionalChatOptions = Omit<
|
||||
>;
|
||||
|
||||
export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
model:
|
||||
| ChatModel
|
||||
// string & {} is a hack to allow any string, but still give autocomplete
|
||||
| (string & {});
|
||||
// Per completion OpenAI params
|
||||
model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
|
||||
temperature: number;
|
||||
topP: number;
|
||||
maxTokens?: number;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { type BaseReader, Document } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type {
|
||||
BaseServiceParams,
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
TranscriptSentence,
|
||||
} from "assemblyai";
|
||||
import { AssemblyAI } from "assemblyai";
|
||||
import type { BaseReader } from "./type.js";
|
||||
|
||||
type AssemblyAIOptions = Partial<BaseServiceParams>;
|
||||
const defaultOptions = {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { type BaseReader, Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import type { ParseConfig } from "papaparse";
|
||||
import Papa from "papaparse";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* papaparse-based csv parser
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { REST, type RESTOptions } from "@discordjs/rest";
|
||||
import { Document, type BaseReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
|
||||
|
||||
@@ -7,7 +7,7 @@ import { Routes, type APIEmbed, type APIMessage } from "discord-api-types/v10";
|
||||
* Represents a reader for Discord messages using @discordjs/rest
|
||||
* See https://github.com/discordjs/discord.js/tree/main/packages/rest
|
||||
*/
|
||||
export class DiscordReader implements BaseReader {
|
||||
export class DiscordReader {
|
||||
private client: REST;
|
||||
|
||||
constructor(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import mammoth from "mammoth";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
export class DocxReader extends FileReader {
|
||||
/** DocxParser */
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* Extract the significant text from an arbitrary HTML document.
|
||||
* The contents of any head, script, style, and xml tags are removed completely.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader, ImageDocument } from "@llamaindex/core/schema";
|
||||
import { ImageDocument } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* Reads the content of an image file into a Document object (which stores the image file as a Blob).
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { JSONValue } from "@llamaindex/core/global";
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
export interface JSONReaderOptions {
|
||||
/**
|
||||
* Whether to ensure only ASCII characters.
|
||||
@@ -183,7 +185,7 @@ export class JSONReader<T extends JSONValue> extends FileReader {
|
||||
return jsonStr;
|
||||
} catch (e) {
|
||||
throw new JSONStringifyError(
|
||||
`Error stringifying JSON: ${e} in "${JSON.stringify(data)}"`,
|
||||
`Error stringifying JSON: ${e} in "${data}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,92 +1,7 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { fs, getEnv } from "@llamaindex/env";
|
||||
import { filetypeinfo } from "magic-bytes.js";
|
||||
|
||||
export type ResultType = "text" | "markdown" | "json";
|
||||
export type Language =
|
||||
| "abq"
|
||||
| "ady"
|
||||
| "af"
|
||||
| "ang"
|
||||
| "ar"
|
||||
| "as"
|
||||
| "ava"
|
||||
| "az"
|
||||
| "be"
|
||||
| "bg"
|
||||
| "bh"
|
||||
| "bho"
|
||||
| "bn"
|
||||
| "bs"
|
||||
| "ch_sim"
|
||||
| "ch_tra"
|
||||
| "che"
|
||||
| "cs"
|
||||
| "cy"
|
||||
| "da"
|
||||
| "dar"
|
||||
| "de"
|
||||
| "en"
|
||||
| "es"
|
||||
| "et"
|
||||
| "fa"
|
||||
| "fr"
|
||||
| "ga"
|
||||
| "gom"
|
||||
| "hi"
|
||||
| "hr"
|
||||
| "hu"
|
||||
| "id"
|
||||
| "inh"
|
||||
| "is"
|
||||
| "it"
|
||||
| "ja"
|
||||
| "kbd"
|
||||
| "kn"
|
||||
| "ko"
|
||||
| "ku"
|
||||
| "la"
|
||||
| "lbe"
|
||||
| "lez"
|
||||
| "lt"
|
||||
| "lv"
|
||||
| "mah"
|
||||
| "mai"
|
||||
| "mi"
|
||||
| "mn"
|
||||
| "mr"
|
||||
| "ms"
|
||||
| "mt"
|
||||
| "ne"
|
||||
| "new"
|
||||
| "nl"
|
||||
| "no"
|
||||
| "oc"
|
||||
| "pi"
|
||||
| "pl"
|
||||
| "pt"
|
||||
| "ro"
|
||||
| "ru"
|
||||
| "rs_cyrillic"
|
||||
| "rs_latin"
|
||||
| "sck"
|
||||
| "sk"
|
||||
| "sl"
|
||||
| "sq"
|
||||
| "sv"
|
||||
| "sw"
|
||||
| "ta"
|
||||
| "tab"
|
||||
| "te"
|
||||
| "th"
|
||||
| "tjk"
|
||||
| "tl"
|
||||
| "tr"
|
||||
| "ug"
|
||||
| "uk"
|
||||
| "ur"
|
||||
| "uz"
|
||||
| "vi";
|
||||
import { FileReader, type Language, type ResultType } from "./type.js";
|
||||
|
||||
const SUPPORT_FILE_EXT: string[] = [
|
||||
".pdf",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
type MarkdownTuple = [string | null, string];
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { BaseReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import type { Crawler, CrawlerOptions, Page } from "notion-md-crawler";
|
||||
import { crawler, pageToString } from "notion-md-crawler";
|
||||
import type { BaseReader } from "./type.js";
|
||||
|
||||
type NotionReaderOptions = Pick<CrawlerOptions, "client" | "serializers">;
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* Read the text of a PDF
|
||||
*/
|
||||
export class PDFReader extends FileReader {
|
||||
async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
|
||||
// XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error if a Buffer passed
|
||||
if (content instanceof Buffer) {
|
||||
content = new Uint8Array(content);
|
||||
}
|
||||
const { totalPages, text } = await readPDF(content);
|
||||
return text.map((text, page) => {
|
||||
const metadata = {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { BaseReader, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { path } from "@llamaindex/env";
|
||||
import { walk } from "../storage/FileSystem.js";
|
||||
import { TextFileReader } from "./TextFileReader.js";
|
||||
import type { BaseReader, FileReader } from "./type.js";
|
||||
import pLimit from "./utils.js";
|
||||
|
||||
type ReaderCallback = (
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { PapaCSVReader } from "./CSVReader.js";
|
||||
import { DocxReader } from "./DocxReader.js";
|
||||
@@ -11,6 +10,7 @@ import {
|
||||
type SimpleDirectoryReaderLoadDataParams,
|
||||
} from "./SimpleDirectoryReader.edge.js";
|
||||
import { TextFileReader } from "./TextFileReader.js";
|
||||
import type { FileReader } from "./type.js";
|
||||
|
||||
export const FILE_EXT_TO_READER: Record<string, FileReader> = {
|
||||
txt: new TextFileReader(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Metadata } from "@llamaindex/core/schema";
|
||||
import { type BaseReader, Document } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import type { MongoClient } from "mongodb";
|
||||
import type { BaseReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* Read in from MongoDB
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { FileReader } from "./type.js";
|
||||
|
||||
/**
|
||||
* Read a .txt file
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { Document } from "@llamaindex/core/schema";
|
||||
import { fs, path } from "@llamaindex/env";
|
||||
|
||||
/**
|
||||
* A reader takes imports data into Document objects.
|
||||
*/
|
||||
export interface BaseReader {
|
||||
loadData(...args: unknown[]): Promise<Document[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A FileReader takes file paths and imports data into Document objects.
|
||||
*/
|
||||
export abstract class FileReader implements BaseReader {
|
||||
abstract loadDataAsContent(
|
||||
fileContent: Uint8Array,
|
||||
fileName?: string,
|
||||
): Promise<Document[]>;
|
||||
|
||||
async loadData(filePath: string): Promise<Document[]> {
|
||||
// XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error in PDFReader
|
||||
const fileContent = new Uint8Array(await fs.readFile(filePath));
|
||||
const fileName = path.basename(filePath);
|
||||
const docs = await this.loadDataAsContent(fileContent, fileName);
|
||||
docs.forEach(FileReader.addMetaData(filePath));
|
||||
return docs;
|
||||
}
|
||||
|
||||
static addMetaData(filePath: string) {
|
||||
return (doc: Document, index: number) => {
|
||||
// generate id as loadDataAsContent is only responsible for the content
|
||||
doc.id_ = `${filePath}_${index + 1}`;
|
||||
doc.metadata["file_path"] = path.resolve(filePath);
|
||||
doc.metadata["file_name"] = path.basename(filePath);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// For LlamaParseReader.ts
|
||||
|
||||
export type ResultType = "text" | "markdown" | "json";
|
||||
export type Language =
|
||||
| "abq"
|
||||
| "ady"
|
||||
| "af"
|
||||
| "ang"
|
||||
| "ar"
|
||||
| "as"
|
||||
| "ava"
|
||||
| "az"
|
||||
| "be"
|
||||
| "bg"
|
||||
| "bh"
|
||||
| "bho"
|
||||
| "bn"
|
||||
| "bs"
|
||||
| "ch_sim"
|
||||
| "ch_tra"
|
||||
| "che"
|
||||
| "cs"
|
||||
| "cy"
|
||||
| "da"
|
||||
| "dar"
|
||||
| "de"
|
||||
| "en"
|
||||
| "es"
|
||||
| "et"
|
||||
| "fa"
|
||||
| "fr"
|
||||
| "ga"
|
||||
| "gom"
|
||||
| "hi"
|
||||
| "hr"
|
||||
| "hu"
|
||||
| "id"
|
||||
| "inh"
|
||||
| "is"
|
||||
| "it"
|
||||
| "ja"
|
||||
| "kbd"
|
||||
| "kn"
|
||||
| "ko"
|
||||
| "ku"
|
||||
| "la"
|
||||
| "lbe"
|
||||
| "lez"
|
||||
| "lt"
|
||||
| "lv"
|
||||
| "mah"
|
||||
| "mai"
|
||||
| "mi"
|
||||
| "mn"
|
||||
| "mr"
|
||||
| "ms"
|
||||
| "mt"
|
||||
| "ne"
|
||||
| "new"
|
||||
| "nl"
|
||||
| "no"
|
||||
| "oc"
|
||||
| "pi"
|
||||
| "pl"
|
||||
| "pt"
|
||||
| "ro"
|
||||
| "ru"
|
||||
| "rs_cyrillic"
|
||||
| "rs_latin"
|
||||
| "sck"
|
||||
| "sk"
|
||||
| "sl"
|
||||
| "sq"
|
||||
| "sv"
|
||||
| "sw"
|
||||
| "ta"
|
||||
| "tab"
|
||||
| "te"
|
||||
| "th"
|
||||
| "tjk"
|
||||
| "tl"
|
||||
| "tr"
|
||||
| "ug"
|
||||
| "uk"
|
||||
| "ur"
|
||||
| "uz"
|
||||
| "vi";
|
||||
|
||||
@@ -18,4 +18,3 @@ export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";
|
||||
export { QdrantVectorStore } from "./vectorStore/QdrantVectorStore.js";
|
||||
export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore.js";
|
||||
export * from "./vectorStore/types.js";
|
||||
export { WeaviateVectorStore } from "./vectorStore/WeaviateVectorStore.js";
|
||||
|
||||
@@ -273,7 +273,7 @@ export class PGVectorStore
|
||||
const paramIndex = params.length + 1;
|
||||
whereClauses.push(`metadata->>'${filter.key}' = $${paramIndex}`);
|
||||
// TODO: support filter with other operators
|
||||
if (!Array.isArray(filter.value) && filter.value) {
|
||||
if (!Array.isArray(filter.value)) {
|
||||
params.push(filter.value);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -36,7 +36,7 @@ type MetadataValue = Record<string, any>;
|
||||
|
||||
// Mapping of filter operators to metadata filter functions
|
||||
const OPERATOR_TO_FILTER: {
|
||||
[key in FilterOperator]?: (
|
||||
[key in FilterOperator]: (
|
||||
{ key, value }: MetadataFilter,
|
||||
metadata: MetadataValue,
|
||||
) => boolean;
|
||||
@@ -94,20 +94,7 @@ const buildFilterFn = (
|
||||
const queryCondition = condition || "and"; // default to and
|
||||
|
||||
const itemFilterFn = (filter: MetadataFilter): boolean => {
|
||||
if (filter.operator === FilterOperator.IS_EMPTY) {
|
||||
// for `is_empty` operator, return true if the metadata key is not present or the value is empty
|
||||
const value = metadata[filter.key];
|
||||
return (
|
||||
value === undefined ||
|
||||
value === null ||
|
||||
value === "" ||
|
||||
(Array.isArray(value) && value.length === 0)
|
||||
);
|
||||
}
|
||||
if (metadata[filter.key] === undefined) {
|
||||
// for other operators, always return false if the metadata key is not present
|
||||
return false;
|
||||
}
|
||||
if (metadata[filter.key] === undefined) return false; // always return false if the metadata key is not present
|
||||
const metadataLookupFn = OPERATOR_TO_FILTER[filter.operator];
|
||||
if (!metadataLookupFn)
|
||||
throw new Error(`Unsupported operator: ${filter.operator}`);
|
||||
|
||||
@@ -1,339 +0,0 @@
|
||||
/* eslint-disable turbo/no-undeclared-env-vars */
|
||||
import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema";
|
||||
import weaviate, {
|
||||
Filters,
|
||||
type Collection,
|
||||
type DeleteManyOptions,
|
||||
type FilterValue,
|
||||
type WeaviateClient,
|
||||
type WeaviateNonGenericObject,
|
||||
} from "weaviate-client";
|
||||
|
||||
import {
|
||||
VectorStoreBase,
|
||||
VectorStoreQueryMode,
|
||||
type IEmbedModel,
|
||||
type MetadataFilter,
|
||||
type MetadataFilters,
|
||||
type VectorStoreNoEmbedModel,
|
||||
type VectorStoreQuery,
|
||||
type VectorStoreQueryResult,
|
||||
} from "./types.js";
|
||||
import {
|
||||
metadataDictToNode,
|
||||
nodeToMetadata,
|
||||
parseArrayValue,
|
||||
parseNumberValue,
|
||||
} from "./utils.js";
|
||||
|
||||
const NODE_SCHEMA = [
|
||||
{
|
||||
dataType: ["text"],
|
||||
description: "Text property",
|
||||
name: "text",
|
||||
},
|
||||
{
|
||||
dataType: ["text"],
|
||||
description: "The ref_doc_id of the Node",
|
||||
name: "ref_doc_id",
|
||||
},
|
||||
{
|
||||
dataType: ["text"],
|
||||
description: "node_info (in JSON)",
|
||||
name: "node_info",
|
||||
},
|
||||
{
|
||||
dataType: ["text"],
|
||||
description: "The relationships of the node (in JSON)",
|
||||
name: "relationships",
|
||||
},
|
||||
];
|
||||
|
||||
const SIMILARITY_KEYS: {
|
||||
[key: string]: "distance" | "score";
|
||||
} = {
|
||||
[VectorStoreQueryMode.DEFAULT]: "distance",
|
||||
[VectorStoreQueryMode.HYBRID]: "score",
|
||||
};
|
||||
|
||||
const buildFilterItem = (
|
||||
collection: Collection,
|
||||
filter: MetadataFilter,
|
||||
): FilterValue => {
|
||||
const { key, operator, value } = filter;
|
||||
|
||||
switch (operator) {
|
||||
case "==": {
|
||||
return collection.filter.byProperty(key).equal(value);
|
||||
}
|
||||
case "!=": {
|
||||
return collection.filter.byProperty(key).notEqual(value);
|
||||
}
|
||||
case ">": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.greaterThan(parseNumberValue(value));
|
||||
}
|
||||
case "<": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.lessThan(parseNumberValue(value));
|
||||
}
|
||||
case ">=": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.greaterOrEqual(parseNumberValue(value));
|
||||
}
|
||||
case "<=": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.lessOrEqual(parseNumberValue(value));
|
||||
}
|
||||
case "any": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.containsAny(parseArrayValue(value).map(String));
|
||||
}
|
||||
case "all": {
|
||||
return collection.filter
|
||||
.byProperty(key)
|
||||
.containsAll(parseArrayValue(value).map(String));
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Operator ${operator} is not supported.`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const toWeaviateFilter = (
|
||||
collection: Collection,
|
||||
standardFilters?: MetadataFilters,
|
||||
): FilterValue | undefined => {
|
||||
if (!standardFilters?.filters.length) return undefined;
|
||||
const filtersList = standardFilters.filters.map((filter) =>
|
||||
buildFilterItem(collection, filter),
|
||||
);
|
||||
if (filtersList.length === 1) return filtersList[0];
|
||||
const condition = standardFilters.condition ?? "and";
|
||||
return Filters[condition](...filtersList);
|
||||
};
|
||||
|
||||
export class WeaviateVectorStore
|
||||
extends VectorStoreBase
|
||||
implements VectorStoreNoEmbedModel
|
||||
{
|
||||
public storesText: boolean = true;
|
||||
private flatMetadata: boolean = true;
|
||||
|
||||
private weaviateClient?: WeaviateClient;
|
||||
private clusterURL!: string;
|
||||
private apiKey!: string;
|
||||
private indexName: string;
|
||||
|
||||
private idKey: string;
|
||||
private contentKey: string;
|
||||
private embeddingKey: string;
|
||||
private metadataKey: string;
|
||||
|
||||
constructor(
|
||||
init?: Partial<IEmbedModel> & {
|
||||
weaviateClient?: WeaviateClient;
|
||||
cloudOptions?: {
|
||||
clusterURL?: string;
|
||||
apiKey?: string;
|
||||
};
|
||||
indexName?: string;
|
||||
idKey?: string;
|
||||
contentKey?: string;
|
||||
metadataKey?: string;
|
||||
embeddingKey?: string;
|
||||
},
|
||||
) {
|
||||
super(init?.embedModel);
|
||||
|
||||
if (init?.weaviateClient) {
|
||||
// Use the provided client
|
||||
this.weaviateClient = init.weaviateClient;
|
||||
} else {
|
||||
// Load client cloud options from config or env
|
||||
const clusterURL =
|
||||
init?.cloudOptions?.clusterURL ?? process.env.WEAVIATE_CLUSTER_URL;
|
||||
const apiKey = init?.cloudOptions?.apiKey ?? process.env.WEAVIATE_API_KEY;
|
||||
if (!clusterURL || !apiKey) {
|
||||
throw new Error(
|
||||
"Must specify WEAVIATE_CLUSTER_URL and WEAVIATE_API_KEY via env variable.",
|
||||
);
|
||||
}
|
||||
this.clusterURL = clusterURL;
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
this.checkIndexName(init?.indexName);
|
||||
this.indexName = init?.indexName ?? "LlamaIndex";
|
||||
this.idKey = init?.idKey ?? "id";
|
||||
this.contentKey = init?.contentKey ?? "text";
|
||||
this.embeddingKey = init?.embeddingKey ?? "vectors";
|
||||
this.metadataKey = init?.metadataKey ?? "node_info";
|
||||
}
|
||||
|
||||
public client() {
|
||||
return this.getClient();
|
||||
}
|
||||
|
||||
public async add(nodes: BaseNode<Metadata>[]): Promise<string[]> {
|
||||
const collection = await this.ensureCollection({ createIfNotExists: true });
|
||||
|
||||
const result = await collection.data.insertMany(
|
||||
nodes.map((node) => {
|
||||
const metadata = nodeToMetadata(
|
||||
node,
|
||||
true,
|
||||
this.contentKey,
|
||||
this.flatMetadata,
|
||||
);
|
||||
const body = {
|
||||
[this.idKey]: node.id_,
|
||||
[this.embeddingKey]: node.getEmbedding(),
|
||||
properties: {
|
||||
...metadata,
|
||||
[this.contentKey]: node.getContent(MetadataMode.NONE),
|
||||
[this.metadataKey]: JSON.stringify(metadata),
|
||||
relationships: JSON.stringify({ ref_doc_id: metadata.ref_doc_id }),
|
||||
},
|
||||
};
|
||||
return body;
|
||||
}),
|
||||
);
|
||||
|
||||
return Object.values(result.uuids);
|
||||
}
|
||||
|
||||
public async delete(
|
||||
refDocId: string,
|
||||
deleteOptions?: DeleteManyOptions<boolean>,
|
||||
): Promise<void> {
|
||||
const collection = await this.ensureCollection();
|
||||
await collection.data.deleteMany(
|
||||
collection.filter.byProperty("ref_doc_id").like(refDocId),
|
||||
deleteOptions,
|
||||
);
|
||||
}
|
||||
|
||||
public async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> {
|
||||
const collection = await this.ensureCollection();
|
||||
const allProperties = await this.getAllProperties();
|
||||
|
||||
let filters: FilterValue | undefined = undefined;
|
||||
|
||||
if (query.docIds) {
|
||||
filters = collection.filter
|
||||
.byProperty("doc_id")
|
||||
.containsAny(query.docIds);
|
||||
}
|
||||
|
||||
if (query.filters) {
|
||||
filters = toWeaviateFilter(collection, query.filters);
|
||||
}
|
||||
|
||||
const queryResult = await collection.query.hybrid(query.queryStr!, {
|
||||
vector: query.queryEmbedding,
|
||||
alpha: this.getQueryAlpha(query),
|
||||
limit: query.similarityTopK,
|
||||
returnMetadata: Object.values(SIMILARITY_KEYS),
|
||||
returnProperties: allProperties,
|
||||
includeVector: true,
|
||||
filters,
|
||||
});
|
||||
|
||||
const entries = queryResult.objects;
|
||||
|
||||
const similarityKey = SIMILARITY_KEYS[query.mode];
|
||||
const nodes: BaseNode<Metadata>[] = [];
|
||||
const similarities: number[] = [];
|
||||
const ids: string[] = [];
|
||||
|
||||
entries.forEach((entry, index) => {
|
||||
if (index < query.similarityTopK && entry.metadata) {
|
||||
const node = metadataDictToNode(entry.properties);
|
||||
node.setContent(entry.properties[this.contentKey]);
|
||||
nodes.push(node);
|
||||
ids.push(entry.uuid);
|
||||
similarities.push(this.getNodeSimilarity(entry, similarityKey));
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
nodes,
|
||||
similarities,
|
||||
ids,
|
||||
};
|
||||
}
|
||||
|
||||
private async getClient(): Promise<WeaviateClient> {
|
||||
if (this.weaviateClient) return this.weaviateClient;
|
||||
const client = await weaviate.connectToWeaviateCloud(this.clusterURL, {
|
||||
authCredentials: new weaviate.ApiKey(this.apiKey),
|
||||
});
|
||||
this.weaviateClient = client;
|
||||
return client;
|
||||
}
|
||||
|
||||
private async ensureCollection({ createIfNotExists = false } = {}) {
|
||||
const client = await this.getClient();
|
||||
const exists = await this.doesCollectionExist();
|
||||
if (!exists) {
|
||||
if (createIfNotExists) {
|
||||
await this.createCollection();
|
||||
} else {
|
||||
throw new Error(`Collection ${this.indexName} does not exist.`);
|
||||
}
|
||||
}
|
||||
return client.collections.get(this.indexName);
|
||||
}
|
||||
|
||||
private async doesCollectionExist() {
|
||||
const client = await this.getClient();
|
||||
return client.collections.exists(this.indexName);
|
||||
}
|
||||
|
||||
private async createCollection() {
|
||||
const client = await this.getClient();
|
||||
return await client.collections.createFromSchema({
|
||||
class: this.indexName,
|
||||
description: `Collection for ${this.indexName}`,
|
||||
properties: NODE_SCHEMA,
|
||||
});
|
||||
}
|
||||
|
||||
private getQueryAlpha(query: VectorStoreQuery): number | undefined {
|
||||
if (!query.queryEmbedding) return undefined;
|
||||
if (query.mode === VectorStoreQueryMode.DEFAULT) return 1;
|
||||
if (query.mode === VectorStoreQueryMode.HYBRID && query.queryStr)
|
||||
return query.alpha;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private async getAllProperties(): Promise<string[]> {
|
||||
const collection = await this.ensureCollection();
|
||||
const properties = (await collection.config.get()).properties;
|
||||
return properties.map((p) => p.name);
|
||||
}
|
||||
|
||||
private checkIndexName(indexName?: string) {
|
||||
if (indexName && indexName[0] !== indexName[0].toUpperCase()) {
|
||||
throw new Error(
|
||||
"Index name must start with a capital letter, e.g. 'LlamaIndex'",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private getNodeSimilarity(
|
||||
entry: WeaviateNonGenericObject,
|
||||
similarityKey: "distance" | "score" = "distance",
|
||||
): number {
|
||||
const distance = entry.metadata?.[similarityKey];
|
||||
if (distance === undefined) return 1;
|
||||
// convert distance https://forum.weaviate.io/t/distance-vs-certainty-scores/258
|
||||
return 1 - distance;
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,6 @@ export enum FilterOperator {
|
||||
ALL = "all", // Contains all (array of strings)
|
||||
TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
|
||||
CONTAINS = "contains", // metadata array contains value (string or number)
|
||||
IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
|
||||
}
|
||||
|
||||
export enum FilterCondition {
|
||||
@@ -45,7 +44,7 @@ export type MetadataFilterValue = string | number | string[] | number[];
|
||||
|
||||
export interface MetadataFilter {
|
||||
key: string;
|
||||
value?: MetadataFilterValue;
|
||||
value: MetadataFilterValue;
|
||||
operator: `${FilterOperator}`; // ==, any, all,...
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ export function metadataDictToNode(
|
||||
}
|
||||
|
||||
export const parsePrimitiveValue = (
|
||||
value?: MetadataFilterValue,
|
||||
value: MetadataFilterValue,
|
||||
): string | number => {
|
||||
if (typeof value !== "number" && typeof value !== "string") {
|
||||
throw new Error("Value must be a string or number");
|
||||
@@ -89,7 +89,7 @@ export const parsePrimitiveValue = (
|
||||
};
|
||||
|
||||
export const parseArrayValue = (
|
||||
value?: MetadataFilterValue,
|
||||
value: MetadataFilterValue,
|
||||
): string[] | number[] => {
|
||||
const isPrimitiveArray =
|
||||
Array.isArray(value) &&
|
||||
@@ -99,8 +99,3 @@ export const parseArrayValue = (
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
export const parseNumberValue = (value?: MetadataFilterValue): number => {
|
||||
if (typeof value !== "number") throw new Error("Value must be a number");
|
||||
return value;
|
||||
};
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
# @llamaindex/core-test
|
||||
|
||||
## 0.0.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 01c184c: Add is_empty operator for filtering vector store
|
||||
|
||||
## 0.0.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
import { expect, test } from "vitest";
|
||||
|
||||
test("Node classes should be included in the top level", async () => {
|
||||
const { Document, IndexNode, TextNode, BaseNode } = await import(
|
||||
"llamaindex"
|
||||
);
|
||||
expect(Document).toBeDefined();
|
||||
expect(IndexNode).toBeDefined();
|
||||
expect(TextNode).toBeDefined();
|
||||
expect(BaseNode).toBeDefined();
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@llamaindex/llamaindex-test",
|
||||
"private": true,
|
||||
"version": "0.0.7",
|
||||
"version": "0.0.6",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "vitest run"
|
||||
|
||||
@@ -256,18 +256,6 @@ describe("SimpleVectorStore", () => {
|
||||
},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
title: "Filter IS_EMPTY",
|
||||
filters: {
|
||||
filters: [
|
||||
{
|
||||
key: "not-exist-key",
|
||||
operator: "is_empty",
|
||||
},
|
||||
],
|
||||
},
|
||||
expected: 3,
|
||||
},
|
||||
{
|
||||
title: "Filter OR",
|
||||
filters: {
|
||||
|
||||
Generated
+149
-483
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user