openai and anthropic upgrade

add changesets

.changeset/bright-eagles-pump.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+CJK sentence splitting (thanks @TomPenguin)

.changeset/good-ghosts-suffer.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Export options for Windows formatted text files

.changeset/happy-jokes-approve.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Disable long sentence splitting by default

.changeset/tough-kings-approve.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Make sentence splitter not split on decimals.

.changeset/wet-cooks-buy.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Anthropic 0.6.1 and OpenAI 4.2.0. Changed Anthropic timeout back to 60s

apps/docs/docusaurus.config.js

@@ -139,6 +139,8 @@ const config = {
         entryPoints: ["../../packages/core/src/index.ts"],
         tsconfig: "../../packages/core/tsconfig.json",
         readme: "none",
+        sourceLinkTemplate:
+          "https://github.com/run-llama/LlamaIndexTS/blob/{gitRevision}/{path}#L{line}",
         sidebar: {
           position: 6,
         },

apps/simple/CHANGELOG.md

@@ -1,5 +1,14 @@
 # simple
 
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [f7a57ca]
+- Updated dependencies [0a09de2]
+- Updated dependencies [f7a57ca]
+  - llamaindex@0.0.21
+
 ## 0.0.18
 
 ### Patch Changes

apps/simple/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.18",
+  "version": "0.0.19",
   "private": true,
   "name": "simple",
   "dependencies": {

package.json

@@ -11,16 +11,16 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.12",
-    "@types/jest": "^29.5.3",
+    "@turbo/gen": "^1.10.13",
+    "@types/jest": "^29.5.4",
     "eslint": "^7.32.0",
     "eslint-config-custom": "workspace:*",
     "husky": "^8.0.3",
-    "jest": "^29.6.2",
+    "jest": "^29.6.3",
     "prettier": "^3.0.2",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.12"
+    "turbo": "^1.10.13"
   },
   "packageManager": "pnpm@7.15.0",
   "dependencies": {

packages/core/CHANGELOG.md

@@ -1,5 +1,13 @@
 # llamaindex
 
+## 0.0.21
+
+### Patch Changes
+
+- f7a57ca: Fixed metadata deserialization (thanks @marcagve)
+- 0a09de2: Update to OpenAI 4.1.0
+- f7a57ca: ChatGPT optimized prompts (thanks @LoganMarkewich)
+
 ## 0.0.20
 
 ### Patch Changes

packages/core/package.json

@@ -1,10 +1,10 @@
 {
   "name": "llamaindex",
-  "version": "0.0.20",
+  "version": "0.0.21",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.6.0",
+    "@anthropic-ai/sdk": "^0.6.1",
     "lodash": "^4.17.21",
-    "openai": "^4.0.1",
+    "openai": "^4.2.0",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
     "replicate": "^0.16.1",
@@ -14,8 +14,8 @@
   },
   "devDependencies": {
     "@types/lodash": "^4.14.197",
-    "@types/node": "^18.17.6",
-    "@types/papaparse": "^5.3.7",
+    "@types/node": "^18.17.9",
+    "@types/papaparse": "^5.3.8",
     "@types/pdf-parse": "^1.1.1",
     "@types/uuid": "^9.0.2",
     "node-stdlib-browser": "^1.2.0",

packages/core/src/NodeParser.ts

@@ -10,7 +10,7 @@ import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";
  */
 export function getTextSplitsFromDocument(
   document: Document,
-  textSplitter: SentenceSplitter
+  textSplitter: SentenceSplitter,
 ) {
   const text = document.getText();
   const splits = textSplitter.splitText(text);
@@ -30,7 +30,7 @@ export function getNodesFromDocument(
   document: Document,
   textSplitter: SentenceSplitter,
   includeMetadata: boolean = true,
-  includePrevNextRel: boolean = true
+  includePrevNextRel: boolean = true,
 ) {
   let nodes: TextNode[] = [];
 
@@ -100,10 +100,10 @@ export class SimpleNodeParser implements NodeParser {
   }) {
     this.textSplitter =
       init?.textSplitter ??
-      new SentenceSplitter(
-        init?.chunkSize ?? DEFAULT_CHUNK_SIZE,
-        init?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP
-      );
+      new SentenceSplitter({
+        chunkSize: init?.chunkSize ?? DEFAULT_CHUNK_SIZE,
+        chunkOverlap: init?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP,
+      });
     this.includeMetadata = init?.includeMetadata ?? true;
     this.includePrevNextRel = init?.includePrevNextRel ?? true;
   }

packages/core/src/Prompt.ts

@@ -11,12 +11,14 @@ export type SimplePrompt = (input: Record<string, string>) => string;
 
 /*
 DEFAULT_TEXT_QA_PROMPT_TMPL = (
-    "Context information is below. \n"
+    "Context information is below.\n"
+    "---------------------\n"
+    "{context_str}\n"
     "---------------------\n"
-    "{context_str}"
-    "\n---------------------\n"
     "Given the context information and not prior knowledge, "
-    "answer the question: {query_str}\n"
+    "answer the query.\n"
+    "Query: {query_str}\n"
+    "Answer: "
 )
 */
 
@@ -27,8 +29,9 @@ export const defaultTextQaPrompt: SimplePrompt = (input) => {
 ---------------------
 ${context}
 ---------------------
-Given the context information and not prior knowledge, answer the question: ${query}
-`;
+Given the context information and not prior knowledge, answer the query.
+Query: ${query}
+Answer:`;
 };
 
 /*
@@ -60,7 +63,7 @@ SUMMARY:"""
 
 /*
 DEFAULT_REFINE_PROMPT_TMPL = (
-    "The original question is as follows: {query_str}\n"
+    "The original query is as follows: {query_str}\n"
     "We have provided an existing answer: {existing_answer}\n"
     "We have the opportunity to refine the existing answer "
     "(only if needed) with some more context below.\n"
@@ -68,21 +71,48 @@ DEFAULT_REFINE_PROMPT_TMPL = (
     "{context_msg}\n"
     "------------\n"
     "Given the new context, refine the original answer to better "
-    "answer the question. "
-    "If the context isn't useful, return the original answer."
+    "answer the query. "
+    "If the context isn't useful, return the original answer.\n"
+    "Refined Answer: "
 )
 */
 
 export const defaultRefinePrompt: SimplePrompt = (input) => {
   const { query = "", existingAnswer = "", context = "" } = input;
 
-  return `The original question is as follows: ${query}
+  return `The original query is as follows: ${query}
 We have provided an existing answer: ${existingAnswer}
 We have the opportunity to refine the existing answer (only if needed) with some more context below.
 ------------
 ${context}
 ------------
-Given the new context, refine the original answer to better answer the question. If the context isn't useful, return the original answer.`;
+Given the new context, refine the original answer to better answer the query. If the context isn't useful, return the original answer.
+Refined Answer:`;
+};
+
+/*
+DEFAULT_TREE_SUMMARIZE_TMPL = (
+  "Context information from multiple sources is below.\n"
+  "---------------------\n"
+  "{context_str}\n"
+  "---------------------\n"
+  "Given the information from multiple sources and not prior knowledge, "
+  "answer the query.\n"
+  "Query: {query_str}\n"
+  "Answer: "
+)
+*/
+
+export const defaultTreeSummarizePrompt: SimplePrompt = (input) => {
+  const { context = "", query = "" } = input;
+
+  return `Context information from multiple sources is below.
+---------------------
+${context}
+---------------------
+Given the information from multiple sources and not prior knowledge, answer the query.
+Query: ${query}
+Answer:`;
 };
 
 export const defaultChoiceSelectPrompt: SimplePrompt = (input) => {

packages/core/src/PromptHelper.ts

@@ -2,9 +2,9 @@ import { globalsHelper } from "./GlobalsHelper";
 import { SimplePrompt } from "./Prompt";
 import { SentenceSplitter } from "./TextSplitter";
 import {
+  DEFAULT_CHUNK_OVERLAP_RATIO,
   DEFAULT_CONTEXT_WINDOW,
   DEFAULT_NUM_OUTPUTS,
-  DEFAULT_CHUNK_OVERLAP_RATIO,
   DEFAULT_PADDING,
 } from "./constants";
 
@@ -43,7 +43,7 @@ export class PromptHelper {
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
     tokenizer?: (text: string) => number[],
-    separator = " "
+    separator = " ",
   ) {
     this.contextWindow = contextWindow;
     this.numOutput = numOutput;
@@ -76,7 +76,7 @@ export class PromptHelper {
   private getAvailableChunkSize(
     prompt: SimplePrompt,
     numChunks = 1,
-    padding = 5
+    padding = 5,
   ) {
     const availableContextSize = this.getAvailableContextSize(prompt);
 
@@ -99,14 +99,14 @@ export class PromptHelper {
   getTextSplitterGivenPrompt(
     prompt: SimplePrompt,
     numChunks = 1,
-    padding = DEFAULT_PADDING
+    padding = DEFAULT_PADDING,
   ) {
     const chunkSize = this.getAvailableChunkSize(prompt, numChunks, padding);
     if (chunkSize === 0) {
       throw new Error("Got 0 as available chunk size");
     }
     const chunkOverlap = this.chunkOverlapRatio * chunkSize;
-    const textSplitter = new SentenceSplitter(chunkSize, chunkOverlap);
+    const textSplitter = new SentenceSplitter({ chunkSize, chunkOverlap });
     return textSplitter;
   }
 
@@ -120,7 +120,7 @@ export class PromptHelper {
   repack(
     prompt: SimplePrompt,
     textChunks: string[],
-    padding = DEFAULT_PADDING
+    padding = DEFAULT_PADDING,
   ) {
     const textSplitter = this.getTextSplitterGivenPrompt(prompt, 1, padding);
     const combinedStr = textChunks.join("\n\n");

packages/core/src/ResponseSynthesizer.ts

@@ -3,6 +3,7 @@ import {
   SimplePrompt,
   defaultRefinePrompt,
   defaultTextQaPrompt,
+  defaultTreeSummarizePrompt,
 } from "./Prompt";
 import { getBiggestPrompt } from "./PromptHelper";
 import { Response } from "./Response";
@@ -35,7 +36,7 @@ interface BaseResponseBuilder {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string>;
 }
 
@@ -54,7 +55,7 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
   async getResponse(
     query: string,
     textChunks: string[],
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
     const input = {
       query,
@@ -78,7 +79,7 @@ export class Refine implements BaseResponseBuilder {
   constructor(
     serviceContext: ServiceContext,
     textQATemplate?: SimplePrompt,
-    refineTemplate?: SimplePrompt
+    refineTemplate?: SimplePrompt,
   ) {
     this.serviceContext = serviceContext;
     this.textQATemplate = textQATemplate ?? defaultTextQaPrompt;
@@ -89,7 +90,7 @@ export class Refine implements BaseResponseBuilder {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string> {
     let response: string | undefined = undefined;
 
@@ -101,7 +102,7 @@ export class Refine implements BaseResponseBuilder {
           prevResponse,
           query,
           chunk,
-          parentEvent
+          parentEvent,
         );
       }
       prevResponse = response;
@@ -113,7 +114,7 @@ export class Refine implements BaseResponseBuilder {
   private async giveResponseSingle(
     queryStr: string,
     textChunk: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: queryStr });
@@ -130,7 +131,7 @@ export class Refine implements BaseResponseBuilder {
             textQATemplate({
               context: chunk,
             }),
-            parentEvent
+            parentEvent,
           )
         ).message.content;
       } else {
@@ -138,7 +139,7 @@ export class Refine implements BaseResponseBuilder {
           response,
           queryStr,
           chunk,
-          parentEvent
+          parentEvent,
         );
       }
     }
@@ -150,7 +151,7 @@ export class Refine implements BaseResponseBuilder {
     response: string,
     queryStr: string,
     textChunk: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ) {
     const refineTemplate: SimplePrompt = (input) =>
       this.refineTemplate({ ...input, query: queryStr });
@@ -166,7 +167,7 @@ export class Refine implements BaseResponseBuilder {
             context: chunk,
             existingAnswer: response,
           }),
-          parentEvent
+          parentEvent,
         )
       ).message.content;
     }
@@ -182,7 +183,7 @@ export class CompactAndRefine extends Refine {
     query: string,
     textChunks: string[],
     parentEvent?: Event,
-    prevResponse?: string
+    prevResponse?: string,
   ): Promise<string> {
     const textQATemplate: SimplePrompt = (input) =>
       this.textQATemplate({ ...input, query: query });
@@ -192,13 +193,13 @@ export class CompactAndRefine extends Refine {
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
     const newTexts = this.serviceContext.promptHelper.repack(
       maxPrompt,
-      textChunks
+      textChunks,
     );
     const response = super.getResponse(
       query,
       newTexts,
       parentEvent,
-      prevResponse
+      prevResponse,
     );
     return response;
   }
@@ -216,10 +217,9 @@ export class TreeSummarize implements BaseResponseBuilder {
   async getResponse(
     query: string,
     textChunks: string[],
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<string> {
-    const summaryTemplate: SimplePrompt = (input) =>
-      defaultTextQaPrompt({ ...input, query: query });
+    const summaryTemplate: SimplePrompt = defaultTreeSummarizePrompt;
 
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
@@ -227,7 +227,7 @@ export class TreeSummarize implements BaseResponseBuilder {
 
     const packedTextChunks = this.serviceContext.promptHelper.repack(
       summaryTemplate,
-      textChunks
+      textChunks,
     );
 
     if (packedTextChunks.length === 1) {
@@ -236,7 +236,7 @@ export class TreeSummarize implements BaseResponseBuilder {
           summaryTemplate({
             context: packedTextChunks[0],
           }),
-          parentEvent
+          parentEvent,
         )
       ).message.content;
     } else {
@@ -246,14 +246,14 @@ export class TreeSummarize implements BaseResponseBuilder {
             summaryTemplate({
               context: chunk,
             }),
-            parentEvent
-          )
-        )
+            parentEvent,
+          ),
+        ),
       );
 
       return this.getResponse(
         query,
-        summaries.map((s) => s.message.content)
+        summaries.map((s) => s.message.content),
       );
     }
   }
@@ -261,7 +261,7 @@ export class TreeSummarize implements BaseResponseBuilder {
 
 export function getResponseBuilder(
   serviceContext: ServiceContext,
-  responseMode?: ResponseMode
+  responseMode?: ResponseMode,
 ): BaseResponseBuilder {
   switch (responseMode) {
     case ResponseMode.SIMPLE:
@@ -296,16 +296,16 @@ export class ResponseSynthesizer {
 
   async synthesize(query: string, nodes: NodeWithScore[], parentEvent?: Event) {
     let textChunks: string[] = nodes.map((node) =>
-      node.node.getContent(MetadataMode.NONE)
+      node.node.getContent(MetadataMode.NONE),
     );
     const response = await this.responseBuilder.getResponse(
       query,
       textChunks,
-      parentEvent
+      parentEvent,
     );
     return new Response(
       response,
-      nodes.map((node) => node.node)
+      nodes.map((node) => node.node),
     );
   }
 }

packages/core/src/TextSplitter.ts

@@ -1,7 +1,7 @@
 // GitHub translated
 
 import { globalsHelper } from "./GlobalsHelper";
-import { DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP } from "./constants";
+import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";
 
 class TextSplit {
   textChunk: string;
@@ -9,7 +9,7 @@ class TextSplit {
 
   constructor(
     textChunk: string,
-    numCharOverlap: number | undefined = undefined
+    numCharOverlap: number | undefined = undefined,
   ) {
     this.textChunk = textChunk;
     this.numCharOverlap = numCharOverlap;
@@ -18,6 +18,38 @@ class TextSplit {
 
 type SplitRep = { text: string; numTokens: number };
 
+/**
+ * Tokenizes sentences. Suitable for English and most European languages.
+ * @param text
+ * @returns
+ */
+export const englishSentenceTokenizer = (text: string) => {
+  // The first part is a lazy match for any character.
+  return text.match(/.+?[.?!]+[\])'"`’”]*(?:\s|$)|.+/g);
+};
+
+/**
+ * Tokenizes sentences. Suitable for Chinese, Japanese, and Korean.
+ * @param text
+ * @returns
+ */
+export const cjkSentenceTokenizer = (text: string) => {
+  // Accepts english style sentence endings with space and
+  // CJK style sentence endings with no space.
+  return text.match(
+    /.+?[.?!]+[\])'"`’”]*(?:\s|$)|.+?[。？！]+[\])'"`’”]*(?:\s|$)?|.+/g,
+  );
+};
+
+export const unixLineSeparator = "\n";
+export const windowsLineSeparator = "\r\n";
+export const unixParagraphSeparator = unixLineSeparator + unixLineSeparator;
+export const windowsParagraphSeparator =
+  windowsLineSeparator + windowsLineSeparator;
+
+// In theory there's also Mac style \r only, but it's pre-OSX and I don't think
+// many documents will use it.
+
 /**
  * SentenceSplitter is our default text splitter that supports splitting into sentences, paragraphs, or fixed length chunks with overlap.
  *
@@ -29,46 +61,44 @@ export class SentenceSplitter {
   private tokenizer: any;
   private tokenizerDecoder: any;
   private paragraphSeparator: string;
-  private chunkingTokenizerFn: any;
-  // private _callback_manager: any;
+  private chunkingTokenizerFn: (text: string) => RegExpMatchArray | null;
+  private splitLongSentences: boolean;
+
+  constructor(options?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+    tokenizer?: any;
+    tokenizerDecoder?: any;
+    paragraphSeparator?: string;
+    chunkingTokenizerFn?: (text: string) => RegExpMatchArray | null;
+    splitLongSentences?: boolean;
+  }) {
+    const {
+      chunkSize = DEFAULT_CHUNK_SIZE,
+      chunkOverlap = DEFAULT_CHUNK_OVERLAP,
+      tokenizer = null,
+      tokenizerDecoder = null,
+      paragraphSeparator = unixParagraphSeparator,
+      chunkingTokenizerFn = undefined,
+      splitLongSentences = false,
+    } = options ?? {};
 
-  constructor(
-    chunkSize: number = DEFAULT_CHUNK_SIZE,
-    chunkOverlap: number = DEFAULT_CHUNK_OVERLAP,
-    tokenizer: any = null,
-    tokenizerDecoder: any = null,
-    paragraphSeparator: string = "\n\n\n",
-    chunkingTokenizerFn: any = undefined
-    // callback_manager: any = undefined
-  ) {
     if (chunkOverlap > chunkSize) {
       throw new Error(
-        `Got a larger chunk overlap (${chunkOverlap}) than chunk size (${chunkSize}), should be smaller.`
+        `Got a larger chunk overlap (${chunkOverlap}) than chunk size (${chunkSize}), should be smaller.`,
       );
     }
     this.chunkSize = chunkSize;
     this.chunkOverlap = chunkOverlap;
     // this._callback_manager = callback_manager || new CallbackManager([]);
 
-    if (chunkingTokenizerFn == undefined) {
-      // define a callable mapping a string to a list of strings
-      const defaultChunkingTokenizerFn = (text: string) => {
-        var result = text.match(/[^.?!]+[.!?]+[\])'"`’”]*|.+/g);
-        return result;
-      };
-
-      chunkingTokenizerFn = defaultChunkingTokenizerFn;
-    }
-
-    if (tokenizer == undefined || tokenizerDecoder == undefined) {
-      tokenizer = globalsHelper.tokenizer();
-      tokenizerDecoder = globalsHelper.tokenizerDecoder();
-    }
-    this.tokenizer = tokenizer;
-    this.tokenizerDecoder = tokenizerDecoder;
+    this.tokenizer = tokenizer ?? globalsHelper.tokenizer();
+    this.tokenizerDecoder =
+      tokenizerDecoder ?? globalsHelper.tokenizerDecoder();
 
     this.paragraphSeparator = paragraphSeparator;
-    this.chunkingTokenizerFn = chunkingTokenizerFn;
+    this.chunkingTokenizerFn = chunkingTokenizerFn ?? englishSentenceTokenizer;
+    this.splitLongSentences = splitLongSentences;
   }
 
   private getEffectiveChunkSize(extraInfoStr?: string): number {
@@ -79,7 +109,7 @@ export class SentenceSplitter {
       effectiveChunkSize = this.chunkSize - numExtraTokens;
       if (effectiveChunkSize <= 0) {
         throw new Error(
-          "Effective chunk size is non positive after considering extra_info"
+          "Effective chunk size is non positive after considering extra_info",
         );
       }
     } else {
@@ -119,7 +149,12 @@ export class SentenceSplitter {
     // Next we split the text using the chunk tokenizer fn/
     let splits = [];
     for (const parText of paragraphSplits) {
-      let sentenceSplits = this.chunkingTokenizerFn(parText);
+      const sentenceSplits = this.chunkingTokenizerFn(parText);
+
+      if (!sentenceSplits) {
+        continue;
+      }
+
       for (const sentence_split of sentenceSplits) {
         splits.push(sentence_split.trim());
       }
@@ -127,13 +162,28 @@ export class SentenceSplitter {
     return splits;
   }
 
+  /**
+   * Splits sentences into chunks if necessary.
+   *
+   * This isn't great behavior because it can split down the middle of a
+   * word or in non-English split down the middle of a Unicode codepoint
+   * so the splitting is turned off by default. If you need it, please
+   * set the splitLongSentences option to true.
+   * @param sentenceSplits
+   * @param effectiveChunkSize
+   * @returns
+   */
   private processSentenceSplits(
     sentenceSplits: string[],
-    effectiveChunkSize: number
+    effectiveChunkSize: number,
   ): SplitRep[] {
-    // Process sentence splits
-    // Primarily check if any sentences exceed the chunk size. If they don't,
-    // force split by tokenizer
+    if (!this.splitLongSentences) {
+      return sentenceSplits.map((split) => ({
+        text: split,
+        numTokens: this.tokenizer(split).length,
+      }));
+    }
+
     let newSplits: SplitRep[] = [];
     for (const split of sentenceSplits) {
       let splitTokens = this.tokenizer(split);
@@ -143,7 +193,7 @@ export class SentenceSplitter {
       } else {
         for (let i = 0; i < splitLen; i += effectiveChunkSize) {
           const cur_split = this.tokenizerDecoder(
-            splitTokens.slice(i, i + effectiveChunkSize)
+            splitTokens.slice(i, i + effectiveChunkSize),
           );
           newSplits.push({ text: cur_split, numTokens: effectiveChunkSize });
         }
@@ -154,7 +204,7 @@ export class SentenceSplitter {
 
   combineTextSplits(
     newSentenceSplits: SplitRep[],
-    effectiveChunkSize: number
+    effectiveChunkSize: number,
   ): TextSplit[] {
     // go through sentence splits, combine to chunks that are within the chunk size
 
@@ -178,8 +228,8 @@ export class SentenceSplitter {
             curChunkSentences
               .map((sentence) => sentence.text)
               .join(" ")
-              .trim()
-          )
+              .trim(),
+          ),
         );
 
         const lastChunkSentences = curChunkSentences;
@@ -210,8 +260,8 @@ export class SentenceSplitter {
         curChunkSentences
           .map((sentence) => sentence.text)
           .join(" ")
-          .trim()
-      )
+          .trim(),
+      ),
     );
     return docs;
   }
@@ -232,13 +282,13 @@ export class SentenceSplitter {
     // force split by tokenizer
     let newSentenceSplits = this.processSentenceSplits(
       sentenceSplits,
-      effectiveChunkSize
+      effectiveChunkSize,
     );
 
     // combine sentence splits into chunks of text that can then be returned
     let combinedTextSplits = this.combineTextSplits(
       newSentenceSplits,
-      effectiveChunkSize
+      effectiveChunkSize,
     );
 
     return combinedTextSplits;

packages/core/src/llm/LLM.ts

@@ -2,9 +2,9 @@ import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { handleOpenAIStream } from "../callbacks/utility/handleOpenAIStream";
 import {
-  AnthropicSession,
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
+  AnthropicSession,
   getAnthropicSession,
 } from "./anthropic";
 import {
@@ -14,7 +14,7 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure";
-import { getOpenAISession, OpenAISession } from "./openai";
+import { OpenAISession, getOpenAISession } from "./openai";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -471,7 +471,7 @@ export class Anthropic implements LLM {
 
     this.apiKey = init?.apiKey ?? undefined;
     this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? undefined; // Default is 60 seconds
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
     this.session =
       init?.session ??
       getAnthropicSession({

packages/core/src/storage/docStore/utils.ts

@@ -21,12 +21,14 @@ export function jsonToDoc(docDict: Record<string, any>): BaseNode {
       id_: dataDict.id_,
       embedding: dataDict.embedding,
       hash: dataDict.hash,
+      metadata: dataDict.metadata,
     });
   } else if (docType === ObjectType.TEXT) {
     doc = new TextNode({
       text: dataDict.text,
       id_: dataDict.id_,
       hash: dataDict.hash,
+      metadata: dataDict.metadata,
     });
   } else {
     throw new Error(`Unknown doc type: ${docType}`);

packages/core/src/tests/TextSplitter.test.ts

@@ -1,4 +1,4 @@
-import { SentenceSplitter } from "../TextSplitter";
+import { SentenceSplitter, cjkSentenceTokenizer } from "../TextSplitter";
 
 describe("SentenceSplitter", () => {
   test("initializes", () => {
@@ -7,17 +7,11 @@ describe("SentenceSplitter", () => {
   });
 
   test("splits paragraphs w/o effective chunk size", () => {
-    const sentenceSplitter = new SentenceSplitter(
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      "\n"
-    );
+    const sentenceSplitter = new SentenceSplitter({});
     // generate the same line as above but correct syntax errors
     let splits = sentenceSplitter.getParagraphSplits(
-      "This is a paragraph.\nThis is another paragraph.",
-      undefined
+      "This is a paragraph.\n\nThis is another paragraph.",
+      undefined,
     );
     expect(splits).toEqual([
       "This is a paragraph.",
@@ -26,17 +20,13 @@ describe("SentenceSplitter", () => {
   });
 
   test("splits paragraphs with effective chunk size", () => {
-    const sentenceSplitter = new SentenceSplitter(
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      "\n"
-    );
+    const sentenceSplitter = new SentenceSplitter({
+      paragraphSeparator: "\n",
+    });
     // generate the same line as above but correct syntax errors
     let splits = sentenceSplitter.getParagraphSplits(
       "This is a paragraph.\nThis is another paragraph.",
-      1000
+      1000,
     );
     expect(splits).toEqual([
       "This is a paragraph.\nThis is another paragraph.",
@@ -47,7 +37,7 @@ describe("SentenceSplitter", () => {
     const sentenceSplitter = new SentenceSplitter();
     let splits = sentenceSplitter.getSentenceSplits(
       "This is a sentence. This is another sentence.",
-      undefined
+      undefined,
     );
     expect(splits).toEqual([
       "This is a sentence.",
@@ -56,19 +46,48 @@ describe("SentenceSplitter", () => {
   });
 
   test("overall split text", () => {
-    let sentenceSplitter = new SentenceSplitter(5, 0);
+    let sentenceSplitter = new SentenceSplitter({
+      chunkSize: 5,
+      chunkOverlap: 0,
+    });
     let splits = sentenceSplitter.splitText(
-      "This is a sentence. This is another sentence."
+      "This is a sentence. This is another sentence.",
     );
     expect(splits).toEqual([
       "This is a sentence.",
       "This is another sentence.",
     ]);
 
-    sentenceSplitter = new SentenceSplitter(1000);
+    sentenceSplitter = new SentenceSplitter({ chunkSize: 1000 });
     splits = sentenceSplitter.splitText(
-      "This is a sentence. This is another sentence."
+      "This is a sentence. This is another sentence.",
     );
     expect(splits).toEqual(["This is a sentence. This is another sentence."]);
   });
+
+  test("doesn't split decimals", () => {
+    let sentenceSplitter = new SentenceSplitter({
+      chunkSize: 5,
+      chunkOverlap: 0,
+    });
+    let splits = sentenceSplitter.splitText(
+      "This is a sentence. This is another sentence. 1.0",
+    );
+    expect(splits).toEqual([
+      "This is a sentence.",
+      "This is another sentence.",
+      "1.0",
+    ]);
+  });
+
+  test("splits cjk", () => {
+    let sentenceSplitter = new SentenceSplitter({
+      chunkSize: 12,
+      chunkOverlap: 0,
+      chunkingTokenizerFn: cjkSentenceTokenizer,
+    });
+
+    const splits = sentenceSplitter.splitText("这是一个句子！这是另一个句子。");
+    expect(splits).toEqual(["这是一个句子！", "这是另一个句子。"]);
+  });
 });