Compare commits

...

3 Commits

Author SHA1 Message Date
github-actions[bot] bf8cbeb6c5 Release 0.5.18 (#1124)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-08-19 12:53:28 +09:00
Alex Yang e27e7dd054 chore: bump natural to 8.0.1 (#1126) 2024-08-17 07:15:08 -07:00
Thuc Pham 8b66cf4341 feat: support organization id in llamacloud index (#1123)
Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>
2024-08-15 13:51:48 +07:00
32 changed files with 400 additions and 1610 deletions
+7
View File
@@ -1,5 +1,12 @@
# docs
## 0.0.59
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.0.58
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "docs",
"version": "0.0.58",
"version": "0.0.59",
"private": true,
"scripts": {
"docusaurus": "docusaurus",
@@ -1,5 +1,13 @@
# @llamaindex/autotool-02-next-example
## 0.1.43
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
- @llamaindex/autotool@2.0.1
## 0.1.42
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/autotool-02-next-example",
"private": true,
"version": "0.1.42",
"version": "0.1.43",
"scripts": {
"dev": "next dev",
"build": "next build",
+1 -1
View File
@@ -51,7 +51,7 @@
"unplugin": "^1.10.1"
},
"peerDependencies": {
"llamaindex": "^0.5.17",
"llamaindex": "^0.5.18",
"openai": "^4",
"typescript": "^4"
},
+7
View File
@@ -1,5 +1,12 @@
# @llamaindex/community
## 0.0.30
### Patch Changes
- Updated dependencies [e27e7dd]
- @llamaindex/core@0.1.9
## 0.0.29
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/community",
"description": "Community package for LlamaIndexTS",
"version": "0.0.29",
"version": "0.0.30",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/core
## 0.1.9
### Patch Changes
- e27e7dd: chore: bump `natural` to 8.0.1
## 0.1.8
### Patch Changes
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/core",
"type": "module",
"version": "0.1.8",
"version": "0.1.9",
"description": "LlamaIndex Core Module",
"exports": {
"./node-parser": {
@@ -132,7 +132,7 @@
"devDependencies": {
"ajv": "^8.16.0",
"bunchee": "5.3.1",
"natural": "^7.1.0"
"natural": "^8.0.1"
},
"dependencies": {
"@llamaindex/env": "workspace:*",
File diff suppressed because it is too large Load Diff
@@ -1,4 +1,5 @@
declare class SentenceTokenizer {
constructor(abbreviations?: string[]);
tokenize(text: string): string[];
}
@@ -0,0 +1,222 @@
var __getOwnPropNames = Object.getOwnPropertyNames;
var __commonJS = (cb, mod) =>
function __require() {
return (
mod ||
(0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod),
mod.exports
);
};
// lib/natural/tokenizers/tokenizer.js
var require_tokenizer = __commonJS({
"lib/natural/tokenizers/tokenizer.js"(exports, module) {
"use strict";
var Tokenizer = class {
trim(array) {
while (array[array.length - 1] === "") {
array.pop();
}
while (array[0] === "") {
array.shift();
}
return array;
}
};
module.exports = Tokenizer;
},
});
// lib/natural/tokenizers/sentence_tokenizer.js
var require_sentence_tokenizer = __commonJS({
"lib/natural/tokenizers/sentence_tokenizer.js"(exports, module) {
var Tokenizer = require_tokenizer();
var NUM = "NUMBER";
var DELIM = "DELIM";
var URI = "URI";
var ABBREV = "ABBREV";
var DEBUG = false;
function generateUniqueCode(base, index) {
return `{{${base}_${index}}}`;
}
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
var SentenceTokenizer = class extends Tokenizer {
constructor(abbreviations) {
super();
if (abbreviations) {
this.abbreviations = abbreviations;
} else {
this.abbreviations = [];
}
this.replacementMap = null;
this.replacementCounter = 0;
}
replaceUrisWithPlaceholders(text) {
const urlPattern =
/(https?:\/\/\S+|www\.\S+|ftp:\/\/\S+|(mailto:)?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|file:\/\/\S+)/gi;
const modifiedText = text.replace(urlPattern, (match) => {
const placeholder = generateUniqueCode(
URI,
this.replacementCounter++,
);
this.replacementMap.set(placeholder, match);
return placeholder;
});
return modifiedText;
}
replaceAbbreviations(text) {
if (this.abbreviations.length === 0) {
return text;
}
const pattern = new RegExp(
`(${this.abbreviations.map((abbrev) => escapeRegExp(abbrev)).join("|")})`,
"gi",
);
const replacedText = text.replace(pattern, (match) => {
const code = generateUniqueCode(ABBREV, this.replacementCounter++);
this.replacementMap.set(code, match);
return code;
});
return replacedText;
}
replaceDelimitersWithPlaceholders(text) {
const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g;
const modifiedText = text.replace(
delimiterPattern,
(match, p1, p2, p3) => {
const placeholder = generateUniqueCode(
DELIM,
this.replacementCounter++,
);
this.delimiterMap.set(placeholder, p1 + p2 + p3);
return placeholder;
},
);
return modifiedText;
}
splitOnPlaceholders(text, placeholders) {
if (this.delimiterMap.size === 0) {
return [text];
}
const keys = Array.from(this.delimiterMap.keys());
const pattern = new RegExp(`(${keys.map(escapeRegExp).join("|")})`);
const parts = text.split(pattern);
const sentences = [];
for (let i = 0; i < parts.length; i += 2) {
const sentence = parts[i];
const placeholder = parts[i + 1] || "";
sentences.push(sentence + placeholder);
}
return sentences;
}
replaceNumbersWithCode(text) {
const numberPattern = /\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b/g;
const replacedText = text.replace(numberPattern, (match) => {
const code = generateUniqueCode(NUM, this.replacementCounter++);
this.replacementMap.set(code, match);
return code;
});
return replacedText;
}
revertReplacements(text) {
let originalText = text;
for (const [
placeholder,
replacement,
] of this.replacementMap.entries()) {
const pattern = new RegExp(escapeRegExp(placeholder), "g");
originalText = originalText.replace(pattern, replacement);
}
return originalText;
}
revertDelimiters(text) {
let originalText = text;
for (const [placeholder, replacement] of this.delimiterMap.entries()) {
const pattern = new RegExp(escapeRegExp(placeholder), "g");
originalText = originalText.replace(pattern, replacement);
}
return originalText;
}
tokenize(text) {
this.replacementCounter = 0;
this.replacementMap = /* @__PURE__ */ new Map();
this.delimiterMap = /* @__PURE__ */ new Map();
DEBUG &&
console.log(
"---Start of sentence tokenization-----------------------",
);
DEBUG && console.log("Original input: >>>" + text + "<<<");
const result1 = this.replaceAbbreviations(text);
DEBUG &&
console.log(
"Phase 1: replacing abbreviations: " + JSON.stringify(result1),
);
const result2 = this.replaceUrisWithPlaceholders(result1);
DEBUG &&
console.log("Phase 2: replacing URIs: " + JSON.stringify(result2));
const result3 = this.replaceNumbersWithCode(result2);
DEBUG &&
console.log(
"Phase 3: replacing numbers with placeholders: " +
JSON.stringify(result3),
);
const result4 = this.replaceDelimitersWithPlaceholders(result3);
DEBUG &&
console.log(
"Phase 4: replacing delimiters with placeholders: " +
JSON.stringify(result4),
);
const sentences = this.splitOnPlaceholders(result4);
DEBUG &&
console.log(
"Phase 5: splitting into sentences on placeholders: " +
JSON.stringify(sentences),
);
const newSentences = sentences.map((s) => {
const s1 = this.revertReplacements(s);
return this.revertDelimiters(s1);
});
DEBUG &&
console.log(
"Phase 6: replacing back abbreviations, URIs, numbers and delimiters: " +
JSON.stringify(newSentences),
);
const trimmedSentences = this.trim(newSentences);
DEBUG &&
console.log(
"Phase 7: trimming array of empty sentences: " +
JSON.stringify(trimmedSentences),
);
const trimmedSentences2 = trimmedSentences.map((sent) => sent.trim());
DEBUG &&
console.log(
"Phase 8: trimming sentences from surrounding whitespace: " +
JSON.stringify(trimmedSentences2),
);
DEBUG &&
console.log(
"---End of sentence tokenization--------------------------",
);
DEBUG &&
console.log(
"---Replacement map---------------------------------------",
);
DEBUG && console.log([...this.replacementMap.entries()]);
DEBUG &&
console.log(
"---Delimiter map-----------------------------------------",
);
DEBUG && console.log([...this.delimiterMap.entries()]);
DEBUG &&
console.log(
"---------------------------------------------------------",
);
return trimmedSentences2;
}
};
module.exports = SentenceTokenizer;
},
});
export default require_sentence_tokenizer();
+9 -3
View File
@@ -1,5 +1,5 @@
import type { TextSplitter } from "./base";
import SentenceTokenizerNew from "./sentence-tokenizer-parser.js";
import SentenceTokenizer from "./sentence_tokenizer";
export type TextSplitterFn = (text: string) => string[];
@@ -31,11 +31,17 @@ export const splitByChar = (): TextSplitterFn => {
return (text: string) => text.split("");
};
let sentenceTokenizer: SentenceTokenizerNew | null = null;
let sentenceTokenizer: SentenceTokenizer | null = null;
export const splitBySentenceTokenizer = (): TextSplitterFn => {
if (!sentenceTokenizer) {
sentenceTokenizer = new SentenceTokenizerNew();
sentenceTokenizer = new SentenceTokenizer([
"i.e.",
"etc.",
"vs.",
"Inc.",
"A.S.A.P.",
]);
}
const tokenizer = sentenceTokenizer;
return (text: string) => {
+7
View File
@@ -1,5 +1,12 @@
# @llamaindex/experimental
## 0.0.68
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.0.67
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.67",
"version": "0.0.68",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+8
View File
@@ -1,5 +1,13 @@
# llamaindex
## 0.5.18
### Patch Changes
- 8b66cf4: feat: support organization id in llamacloud index
- Updated dependencies [e27e7dd]
- @llamaindex/core@0.1.9
## 0.5.17
### Patch Changes
@@ -1,5 +1,12 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.52
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.0.51
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.51",
"version": "0.0.52",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,12 @@
# @llamaindex/next-agent-test
## 0.1.52
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.1.51
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.51",
"version": "0.1.52",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,12 @@
# test-edge-runtime
## 0.1.51
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.1.50
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.50",
"version": "0.1.51",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,12 @@
# @llamaindex/next-node-runtime
## 0.0.33
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.0.32
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.0.32",
"version": "0.0.33",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,12 @@
# @llamaindex/waku-query-engine-test
## 0.0.52
### Patch Changes
- Updated dependencies [8b66cf4]
- llamaindex@0.5.18
## 0.0.51
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.51",
"version": "0.0.52",
"type": "module",
"private": true,
"scripts": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.5.17",
"version": "0.5.18",
"license": "MIT",
"type": "module",
"keywords": [
@@ -8,7 +8,7 @@ import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
import { getPipelineCreate } from "./config.js";
import type { CloudConstructorParams } from "./constants.js";
import { getAppBaseUrl, initService } from "./utils.js";
import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
import { SentenceSplitter } from "@llamaindex/core/node-parser";
@@ -132,18 +132,28 @@ export class LlamaCloudIndex {
await this.waitForPipelineIngestion(verbose, raiseOnError);
}
private async getPipelineId(
name: string,
projectName: string,
public async getPipelineId(
name?: string,
projectName?: string,
): Promise<string> {
const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
projectName,
pipelineName: name,
projectId: await this.getProjectId(projectName),
pipelineName: name ?? this.params.name,
});
return pipelines[0].id;
}
public async getProjectId(
projectName?: string,
organizationId?: string,
): Promise<string> {
return await getProjectId(
projectName ?? this.params.projectName,
organizationId ?? this.params.organizationId,
);
}
static async fromDocuments(
params: {
documents: Document[];
@@ -168,6 +178,7 @@ export class LlamaCloudIndex {
});
const project = await ProjectsService.upsertProjectApiV1ProjectsPut({
organizationId: params.organizationId,
requestBody: {
name: params.projectName ?? "default",
},
@@ -11,7 +11,7 @@ import { extractText, wrapEventCaller } from "@llamaindex/core/utils";
import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
import type { ClientParams, CloudConstructorParams } from "./constants.js";
import { DEFAULT_PROJECT_NAME } from "./constants.js";
import { initService } from "./utils.js";
import { getProjectId, initService } from "./utils.js";
export type CloudRetrieveParams = Omit<
RetrievalParams,
@@ -21,6 +21,7 @@ export type CloudRetrieveParams = Omit<
export class LlamaCloudRetriever implements BaseRetriever {
clientParams: ClientParams;
retrieveParams: CloudRetrieveParams;
organizationId?: string;
projectName: string = DEFAULT_PROJECT_NAME;
pipelineName: string;
@@ -49,6 +50,9 @@ export class LlamaCloudRetriever implements BaseRetriever {
if (params.projectName) {
this.projectName = params.projectName;
}
if (params.organizationId) {
this.organizationId = params.organizationId;
}
}
@wrapEventCaller
@@ -57,7 +61,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
preFilters,
}: RetrieveParams): Promise<NodeWithScore[]> {
const pipelines = await PipelinesService.searchPipelinesApiV1PipelinesGet({
projectName: this.projectName,
projectId: await getProjectId(this.projectName, this.organizationId),
pipelineName: this.pipelineName,
});
@@ -8,5 +8,6 @@ export type ClientParams = { apiKey?: string; baseUrl?: string };
export type CloudConstructorParams = {
name: string;
projectName: string;
organizationId?: string;
serviceContext?: ServiceContext;
} & ClientParams;
+29 -1
View File
@@ -1,4 +1,4 @@
import { OpenAPI } from "@llamaindex/cloud/api";
import { OpenAPI, ProjectsService } from "@llamaindex/cloud/api";
import { getEnv } from "@llamaindex/env";
import type { ClientParams } from "./constants.js";
import { DEFAULT_BASE_URL } from "./constants.js";
@@ -20,3 +20,31 @@ export function initService({ apiKey, baseUrl }: ClientParams = {}) {
);
}
}
export async function getProjectId(
projectName: string,
organizationId?: string,
): Promise<string> {
const projects = await ProjectsService.listProjectsApiV1ProjectsGet({
projectName: projectName,
organizationId: organizationId,
});
if (projects.length === 0) {
throw new Error(
`Unknown project name ${projectName}. Please confirm a managed project with this name exists.`,
);
} else if (projects.length > 1) {
throw new Error(
`Multiple projects found with name ${projectName}. Please specify organization_id.`,
);
}
const project = projects[0];
if (!project.id) {
throw new Error(`No project found with name ${projectName}`);
}
return project.id;
}
+24 -14
View File
@@ -163,7 +163,7 @@ importers:
version: link:../packages/llamaindex
mongodb:
specifier: ^6.7.0
version: 6.8.0(@aws-sdk/credential-providers@3.613.0)
version: 6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0)))
pathe:
specifier: ^1.1.2
version: 1.1.2
@@ -382,8 +382,8 @@ importers:
specifier: 5.3.1
version: 5.3.1(typescript@5.5.3)
natural:
specifier: ^7.1.0
version: 7.1.0(@aws-sdk/credential-providers@3.613.0)
specifier: ^8.0.1
version: 8.0.1(@aws-sdk/credential-providers@3.613.0)
packages/core/tests:
devDependencies:
@@ -568,7 +568,7 @@ importers:
version: 2.0.0
mongodb:
specifier: ^6.7.0
version: 6.8.0(@aws-sdk/credential-providers@3.613.0)
version: 6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0)))
notion-md-crawler:
specifier: ^1.0.0
version: 1.0.0(encoding@0.1.13)
@@ -8099,8 +8099,8 @@ packages:
natural-compare@1.4.0:
resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
natural@7.1.0:
resolution: {integrity: sha512-GBhiRgF0VUX+zPWahBVir1ajARQDZF1Fe6UpQORNzyQT57JQ2KLKYvubecvjIYh/uDaociusmySeRh+WL5OdxQ==}
natural@8.0.1:
resolution: {integrity: sha512-VVw8O5KrfvwqAFeNZEgBbdgA+AQaBlHcXEootWU7TWDaFWFI0VLfzyKMsRjnfdS3cVCpWmI04xLJonCvEv11VQ==}
engines: {node: '>=0.4.10'}
negotiator@0.6.3:
@@ -12003,7 +12003,7 @@ snapshots:
'@babel/core': 7.24.7
'@babel/helper-compilation-targets': 7.24.7
'@babel/helper-plugin-utils': 7.24.7
debug: 4.3.5
debug: 4.3.6
lodash.debounce: 4.0.8
resolve: 1.22.8
transitivePeerDependencies:
@@ -15604,7 +15604,7 @@ snapshots:
dependencies:
'@typescript-eslint/types': 5.62.0
'@typescript-eslint/visitor-keys': 5.62.0
debug: 4.3.5
debug: 4.3.6
globby: 11.1.0
is-glob: 4.0.3
semver: 7.6.2
@@ -15942,7 +15942,7 @@ snapshots:
agent-base@6.0.2:
dependencies:
debug: 4.3.5
debug: 4.3.6
transitivePeerDependencies:
- supports-color
optional: true
@@ -17742,6 +17742,16 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
eslint-module-utils@2.8.1(@typescript-eslint/parser@7.16.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 7.16.0(eslint@8.57.0)(typescript@5.5.3)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
transitivePeerDependencies:
- supports-color
eslint-module-utils@2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7
@@ -17763,7 +17773,7 @@ snapshots:
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.16.0(eslint@8.57.0)(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0)
hasown: 2.0.2
is-core-module: 2.14.0
is-glob: 4.0.3
@@ -20463,7 +20473,7 @@ snapshots:
optionalDependencies:
'@aws-sdk/credential-providers': 3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))
mongodb@6.8.0(@aws-sdk/credential-providers@3.613.0):
mongodb@6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))):
dependencies:
'@mongodb-js/saslprep': 1.1.7
bson: 6.8.0
@@ -20494,7 +20504,7 @@ snapshots:
mquery@5.0.0:
dependencies:
debug: 4.3.5
debug: 4.3.6
transitivePeerDependencies:
- supports-color
@@ -20532,7 +20542,7 @@ snapshots:
natural-compare@1.4.0: {}
natural@7.1.0(@aws-sdk/credential-providers@3.613.0):
natural@8.0.1(@aws-sdk/credential-providers@3.613.0):
dependencies:
afinn-165: 1.0.4
afinn-165-financialmarketnews: 3.0.0
@@ -22645,7 +22655,7 @@ snapshots:
spdy-transport@3.0.0:
dependencies:
debug: 4.3.5
debug: 4.3.6
detect-node: 2.1.0
hpack.js: 2.1.6
obuf: 1.1.2