Compare commits

..

11 Commits

Author SHA1 Message Date
github-actions[bot] 1752463ee6 Release 0.5.14 (#1103)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-08-07 17:03:00 -07:00
Peter Goldstein c825a2f743 Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure (#1102) 2024-08-06 14:23:28 +07:00
github-actions[bot] ba058dc8d4 Release 0.5.13 (#1100)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-08-02 09:50:07 -07:00
Philipp Serrer 04b2f8e062 fix: metadata should not be included after sentence splitter (#1099) 2024-08-02 09:22:04 -07:00
Alex Yang 62b874e14f fix: enforce no-base-to-string (#1097) 2024-08-01 14:05:19 -07:00
github-actions[bot] 9c9e9b4e03 Release 0.5.12 (#1091)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-07-30 15:46:39 -07:00
Alex Yang e3c307ab55 chore: fix changeset 2024-07-30 15:32:24 -07:00
André Lago b1b2baa969 docs: fix minor typo (#1092) 2024-07-30 15:07:46 -07:00
Marcus Schiesser 0452af91cc fix: handling errors in splitBySentenceTokenizer (#1087)
Co-authored-by: Alex Yang <himself65@outlook.com>
2024-07-30 09:36:58 -07:00
Marcus Schiesser da5cfc42e5 fix: integrate with create-llama (#1088)
Co-authored-by: Alex Yang <himself65@outlook.com>
2024-07-30 08:19:32 -07:00
Alex Yang eb89223386 chore: bump bunchee@5.3.1 (#1090) 2024-07-30 08:19:01 -07:00
42 changed files with 638 additions and 180 deletions
-5
View File
@@ -1,5 +0,0 @@
---
"llamaindex": patch
---
feat: add splitByPage mode to LlamaParseReader
-6
View File
@@ -1,6 +0,0 @@
---
"@llamaindex/core": patch
"@llamaindex/core-tests": patch
---
fix: handling errors in splitBySentenceTokenizer
+6 -1
View File
@@ -31,7 +31,12 @@ module.exports = {
"@typescript-eslint/ban-types": "off",
"no-array-constructor": "off",
"@typescript-eslint/no-array-constructor": "off",
"@typescript-eslint/no-base-to-string": "off",
"@typescript-eslint/no-base-to-string": [
"error",
{
ignoredTypeNames: ["Error", "RegExp", "URL", "URLSearchParams"],
},
],
"@typescript-eslint/no-duplicate-enum-values": "off",
"@typescript-eslint/no-duplicate-type-constituents": "off",
"@typescript-eslint/no-explicit-any": "off",
+1 -1
View File
@@ -164,7 +164,7 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
- [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that question. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
- [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
+22
View File
@@ -1,5 +1,27 @@
# docs
## 0.0.55
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.0.54
### Patch Changes
- llamaindex@0.5.13
## 0.0.53
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.0.52
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "docs",
"version": "0.0.52",
"version": "0.0.55",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
@@ -1,5 +1,30 @@
# @llamaindex/autotool-02-next-example
## 0.1.39
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
- @llamaindex/autotool@2.0.0
## 0.1.38
### Patch Changes
- llamaindex@0.5.13
- @llamaindex/autotool@2.0.0
## 0.1.37
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
- @llamaindex/autotool@2.0.0
## 0.1.36
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool-02-next-example",
"private": true,
"version": "0.1.36",
"version": "0.1.39",
"scripts": {
"dev": "next dev",
"build": "next build",
+2 -2
View File
@@ -51,7 +51,7 @@
"unplugin": "^1.10.1"
},
"peerDependencies": {
"llamaindex": "^0.5.11",
"llamaindex": "^0.5.14",
"openai": "^4",
"typescript": "^4"
},
@@ -70,7 +70,7 @@
"@swc/types": "^0.1.8",
"@types/json-schema": "^7.0.15",
"@types/node": "^20.12.11",
"bunchee": "5.3.0-beta.0",
"bunchee": "5.3.1",
"llamaindex": "workspace:*",
"next": "14.2.5",
"rollup": "^4.18.0",
+1 -1
View File
@@ -35,6 +35,6 @@
},
"devDependencies": {
"@hey-api/openapi-ts": "^0.48.0",
"bunchee": "5.3.0-beta.0"
"bunchee": "5.3.1"
}
}
+14
View File
@@ -1,5 +1,19 @@
# @llamaindex/community
## 0.0.28
### Patch Changes
- Updated dependencies [04b2f8e]
- @llamaindex/core@0.1.7
## 0.0.27
### Patch Changes
- Updated dependencies [0452af9]
- @llamaindex/core@0.1.6
## 0.0.26
### Patch Changes
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/community",
"description": "Community package for LlamaIndexTS",
"version": "0.0.26",
"version": "0.0.28",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
@@ -43,7 +43,7 @@
},
"devDependencies": {
"@types/node": "^20.14.2",
"bunchee": "5.3.0-beta.0"
"bunchee": "5.3.1"
},
"dependencies": {
"@aws-sdk/client-bedrock-runtime": "^3.613.0",
+12
View File
@@ -1,5 +1,17 @@
# @llamaindex/core
## 0.1.7
### Patch Changes
- 04b2f8e: Fix issue with metadata included after sentence splitter
## 0.1.6
### Patch Changes
- 0452af9: fix: handling errors in splitBySentenceTokenizer
## 0.1.5
### Patch Changes
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/core",
"type": "module",
"version": "0.1.5",
"version": "0.1.7",
"description": "LlamaIndex Core Module",
"exports": {
"./node-parser": {
@@ -131,7 +131,7 @@
},
"devDependencies": {
"ajv": "^8.16.0",
"bunchee": "5.3.0-beta.0",
"bunchee": "5.3.1",
"natural": "^7.1.0"
},
"dependencies": {
+1 -1
View File
@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
return nodes.reduce<TextNode[]>((allNodes, node) => {
const metadataStr = this.getMetadataString(node);
const splits = this.splitTextMetadataAware(
node.getContent(MetadataMode.ALL),
node.getContent(MetadataMode.NONE),
metadataStr,
);
return allNodes.concat(buildNodeFromSplits(splits, node));
+22
View File
@@ -1,5 +1,27 @@
# @llamaindex/experimental
## 0.0.64
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.0.63
### Patch Changes
- llamaindex@0.5.13
## 0.0.62
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.0.61
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.61",
"version": "0.0.64",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
@@ -162,18 +162,18 @@ export class JSONQueryEngine implements QueryEngine {
const schema = this.getSchemaContext();
const jsonPathResponseStr = await this.serviceContext.llm.complete({
const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
prompt: this.jsonPathPrompt({ query, schema }),
});
if (this.verbose) {
console.log(
`> JSONPath Instructions:\n\`\`\`\n${jsonPathResponseStr}\n\`\`\`\n`,
`> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
);
}
const jsonPathOutput = await this.outputProcessor({
llmOutput: jsonPathResponseStr.text,
llmOutput: jsonPathResponse,
jsonValue: this.jsonValue,
});
@@ -188,7 +188,7 @@ export class JSONQueryEngine implements QueryEngine {
prompt: this.responseSynthesisPrompt({
query,
jsonSchema: schema,
jsonPath: jsonPathResponseStr.text,
jsonPath: jsonPathResponse,
jsonPathValue: JSON.stringify(jsonPathOutput),
}),
});
@@ -199,7 +199,7 @@ export class JSONQueryEngine implements QueryEngine {
}
const responseMetadata = {
jsonPathResponseStr,
jsonPathResponse,
};
const response = EngineResponse.fromResponse(responseStr, false);
+23
View File
@@ -1,5 +1,28 @@
# llamaindex
## 0.5.14
### Patch Changes
- c825a2f: Add gpt-4o-mini to Azure. Add 2024-06-01 API version for Azure
## 0.5.13
### Patch Changes
- Updated dependencies [04b2f8e]
- @llamaindex/core@0.1.7
## 0.5.12
### Patch Changes
- 345300f: feat: add splitByPage mode to LlamaParseReader
- da5cfc4: Add metadatafilter options to retriever constructors
- da5cfc4: Fix system prompt not used in ContextChatEngine
- Updated dependencies [0452af9]
- @llamaindex/core@0.1.6
## 0.5.11
### Patch Changes
@@ -1,5 +1,27 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.48
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.0.47
### Patch Changes
- llamaindex@0.5.13
## 0.0.46
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.0.45
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.45",
"version": "0.0.48",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,27 @@
# @llamaindex/next-agent-test
## 0.1.48
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.1.47
### Patch Changes
- llamaindex@0.5.13
## 0.1.46
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.1.45
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.45",
"version": "0.1.48",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,27 @@
# test-edge-runtime
## 0.1.47
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.1.46
### Patch Changes
- llamaindex@0.5.13
## 0.1.45
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.1.44
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.44",
"version": "0.1.47",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,27 @@
# @llamaindex/next-node-runtime
## 0.0.29
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.0.28
### Patch Changes
- llamaindex@0.5.13
## 0.0.27
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.0.26
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.0.26",
"version": "0.0.29",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,27 @@
# @llamaindex/waku-query-engine-test
## 0.0.48
### Patch Changes
- Updated dependencies [c825a2f]
- llamaindex@0.5.14
## 0.0.47
### Patch Changes
- llamaindex@0.5.13
## 0.0.46
### Patch Changes
- Updated dependencies [345300f]
- Updated dependencies [da5cfc4]
- Updated dependencies [da5cfc4]
- llamaindex@0.5.12
## 0.0.45
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.45",
"version": "0.0.48",
"type": "module",
"private": true,
"scripts": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.5.11",
"version": "0.5.14",
"license": "MIT",
"type": "module",
"keywords": [
@@ -15,8 +15,8 @@ import { initService } from "./utils.js";
export type CloudRetrieveParams = Omit<
RetrievalParams,
"query" | "searchFilters" | "className" | "denseSimilarityTopK"
> & { similarityTopK?: number };
"query" | "search_filters" | "dense_similarity_top_k"
> & { similarityTopK?: number; filters?: MetadataFilters };
export class LlamaCloudRetriever implements BaseRetriever {
clientParams: ClientParams;
@@ -84,7 +84,9 @@ export class LlamaCloudRetriever implements BaseRetriever {
requestBody: {
...this.retrieveParams,
query: extractText(query),
search_filters: preFilters as MetadataFilters,
search_filters:
this.retrieveParams.filters ?? (preFilters as MetadataFilters),
dense_similarity_top_k: this.retrieveParams.similarityTopK,
},
});
@@ -79,7 +79,7 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
private async getImageInput(
image: ImageType,
): Promise<{ bytes: string } | { url: string }> {
if (isLocal(image)) {
if (isLocal(image) || image instanceof Blob) {
const base64 = await imageToDataUrl(image);
const bytes = base64.split(",")[1];
return { bytes };
@@ -126,7 +126,7 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
if (!this.systemPrompt) return message;
return {
...message,
content: this.systemPrompt.trim() + "\n" + message.content,
content: this.systemPrompt.trim() + "\n" + extractText(message.content),
};
}
}
@@ -133,7 +133,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
const responses: EngineResponse[] = [];
for (let i = 0; i < result.selections.length; i++) {
const engineInd = result.selections[i];
const logStr = `Selecting query engine ${engineInd}: ${result.selections[i]}.`;
const logStr = `Selecting query engine ${engineInd.index}: ${result.selections[i].index}.`;
if (this.verbose) {
console.log(logStr + "\n");
@@ -119,15 +119,15 @@ export class SubQuestionQueryEngine
return null;
}
const responseText = await queryEngine?.call?.({
const responseValue = await queryEngine?.call?.({
query: question,
});
if (!responseText) {
if (responseValue == null) {
return null;
}
const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
const nodeText = `Sub question: ${question}\nResponse: ${typeof responseValue === "string" ? responseValue : JSON.stringify(responseValue)}`;
const node = new TextNode({ text: nodeText });
return { node, score: 0 };
} catch (error) {
@@ -78,7 +78,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
serviceContext: this.serviceContext,
});
const queryResponse = `Question: ${query}\nResponse: ${response}`;
const queryResponse = `Question: ${extractText(query)}\nResponse: ${response}`;
const queryEngine = index.asQueryEngine();
@@ -386,6 +386,7 @@ export type VectorIndexRetrieverOptions = {
index: VectorStoreIndex;
similarityTopK?: number;
topK?: TopKMap;
filters?: MetadataFilters;
};
export class VectorIndexRetriever implements BaseRetriever {
@@ -393,14 +394,21 @@ export class VectorIndexRetriever implements BaseRetriever {
topK: TopKMap;
serviceContext?: ServiceContext;
filters?: MetadataFilters;
constructor({ index, similarityTopK, topK }: VectorIndexRetrieverOptions) {
constructor({
index,
similarityTopK,
topK,
filters,
}: VectorIndexRetrieverOptions) {
this.index = index;
this.serviceContext = this.index.serviceContext;
this.topK = topK ?? {
[ModalityType.TEXT]: similarityTopK ?? DEFAULT_SIMILARITY_TOP_K,
[ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
};
this.filters = filters;
}
/**
@@ -443,7 +451,7 @@ export class VectorIndexRetriever implements BaseRetriever {
query: MessageContent,
type: ModalityType,
vectorStore: VectorStore,
preFilters?: MetadataFilters,
filters?: MetadataFilters,
): Promise<NodeWithScore[]> {
// convert string message to multi-modal format
if (typeof query === "string") {
@@ -460,7 +468,7 @@ export class VectorIndexRetriever implements BaseRetriever {
queryEmbedding,
mode: VectorStoreQueryMode.DEFAULT,
similarityTopK: this.topK[type],
filters: preFilters ?? undefined,
filters: this.filters ?? filters ?? undefined,
});
nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
}
+6
View File
@@ -18,6 +18,7 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
openAIModel: "gpt-3.5-turbo-16k",
},
"gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
"gpt-4o-mini": { contextWindow: 128000, openAIModel: "gpt-4o-mini" },
"gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
"gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
"gpt-4-turbo": {
@@ -40,6 +41,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
contextWindow: 128000,
openAIModel: "gpt-4o-2024-05-13",
},
"gpt-4o-mini-2024-07-18": {
contextWindow: 128000,
openAIModel: "gpt-4o-mini-2024-07-18",
},
};
const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -73,6 +78,7 @@ const ALL_AZURE_API_VERSIONS = [
"2024-03-01-preview",
"2024-04-01-preview",
"2024-05-01-preview",
"2024-06-01",
];
const DEFAULT_API_VERSION = "2023-05-15";
@@ -185,7 +185,7 @@ export class JSONReader<T extends JSONValue> extends FileReader {
return jsonStr;
} catch (e) {
throw new JSONStringifyError(
`Error stringifying JSON: ${e} in "${data}"`,
`Error stringifying JSON: ${e} in "${JSON.stringify(data)}"`,
);
}
}
@@ -6,6 +6,10 @@ import { FileReader } from "./type.js";
*/
export class PDFReader extends FileReader {
async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
// XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error if a Buffer passed
if (content instanceof Buffer) {
content = new Uint8Array(content);
}
const { totalPages, text } = await readPDF(content);
return text.map((text, page) => {
const metadata = {
+1 -2
View File
@@ -18,8 +18,7 @@ export abstract class FileReader implements BaseReader {
): Promise<Document[]>;
async loadData(filePath: string): Promise<Document[]> {
// XXX: create a new Uint8Array to prevent "Please provide binary data as `Uint8Array`, rather than `Buffer`." error in PDFReader
const fileContent = new Uint8Array(await fs.readFile(filePath));
const fileContent = await fs.readFile(filePath);
const fileName = path.basename(filePath);
const docs = await this.loadDataAsContent(fileContent, fileName);
docs.forEach(FileReader.addMetaData(filePath));
+346 -129
View File
File diff suppressed because it is too large Load Diff