fix: ignore error

.changeset/chilled-tomatoes-visit.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: add splitByPage mode to LlamaParseReader

.changeset/thin-pens-deliver.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/core": patch
+"@llamaindex/core-tests": patch
+---
+
+fix: handling errors in splitBySentenceTokenizer

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -48,6 +48,7 @@ They can be divided into two groups.
 - `gpt4oApiKey?` Deprecated. Use vendorMultimodal params. Optional. Set the GPT-4o API key. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
 - `boundingBox?` Optional. Specify an area of the document to parse. Expects the bounding box margins as a string in clockwise order, e.g. `boundingBox = "0.1,0,0,0"` to not parse the top 10% of the document.
 - `targetPages?` Optional. Specify which pages to parse by specifying them as a comma-separated list. First page is `0`.
+- `splitByPage` Wether to split the results, creating one document per page. Uses the set `pageSeparator` or `\n---\n` as fallback. Default is true.
 - `useVendorMultimodalModel` set to true to use a multimodal model. Default is `false`.
 - `vendorMultimodalModel?` Optional. Specify which multimodal model to use. Default is GPT4o. See [here](https://docs.cloud.llamaindex.ai/llamaparse/features/multimodal) for a list of available models and cost.
 - `vendorMultimodalApiKey?` Optional. Set the multimodal model API key. Can also be set in the environment variable `LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY`.

packages/cloud/package.json

@@ -4,7 +4,7 @@
   "type": "module",
   "license": "MIT",
   "scripts": {
-    "generate": "pnpm dlx @hey-api/openapi-ts",
+    "generate": "pnpm dlx @hey-api/openapi-ts@0.49.0",
     "build": "pnpm run generate && bunchee"
   },
   "files": [

packages/core/src/node-parser/utils.ts

@@ -39,7 +39,11 @@ export const splitBySentenceTokenizer = (): TextSplitterFn => {
   }
   const tokenizer = sentenceTokenizer;
   return (text: string) => {
-    return tokenizer.tokenize(text);
+    try {
+      return tokenizer.tokenize(text);
+    } catch {
+      return [text];
+    }
   };
 };
 

packages/core/tests/node-parser/text-splitter.test.ts

@@ -1,7 +1,10 @@
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import {
+  SentenceSplitter,
+  splitBySentenceTokenizer,
+} from "@llamaindex/core/node-parser";
 import { describe, expect, test } from "vitest";
 
-describe("SentenceSplitter", () => {
+describe("sentence splitter", () => {
   test("initializes", () => {
     const sentenceSplitter = new SentenceSplitter();
     expect(sentenceSplitter).toBeDefined();
@@ -105,4 +108,11 @@ describe("SentenceSplitter", () => {
       "因为他照了人类，连我都在内。",
     ]);
   });
+
+  test("issue 1087 - edge case when input with brackets", () => {
+    const text =
+      "A card must be of uniform thickness and made of unfolded and uncreased paper or cardstock of approximately the quality and weight of a stamped card (i.e., a card available from USPS).";
+    const split = splitBySentenceTokenizer();
+    expect(split(text)).toEqual([text]);
+  });
 });

packages/llamaindex/src/readers/LlamaParseReader.ts

@@ -143,6 +143,8 @@ export class LlamaParseReader extends FileReader {
   targetPages?: string;
   // Whether or not to ignore and skip errors raised during parsing.
   ignoreErrors: boolean = true;
+  // Whether to split by page using the pageSeparator or '\n---\n' as default.
+  splitByPage: boolean = true;
   // Whether to use the vendor multimodal API.
   useVendorMultimodalModel: boolean = false;
   // The model name for the vendor multimodal API
@@ -326,10 +328,17 @@ export class LlamaParseReader extends FileReader {
       }
 
       // Return results as Document objects
-      const resultJson = await this.getJobResult(jobId, this.resultType);
+      const jobResults = await this.getJobResult(jobId, this.resultType);
+      const resultText = jobResults[this.resultType];
+
+      // Split the text by separator if splitByPage is true
+      if (this.splitByPage) {
+        return this.splitTextBySeparator(resultText);
+      }
+
       return [
         new Document({
-          text: resultJson[this.resultType],
+          text: resultText,
         }),
       ];
     } catch (e) {
@@ -485,6 +494,17 @@ export class LlamaParseReader extends FileReader {
     return filteredParams;
   }
 
+  private splitTextBySeparator(text: string): Document[] {
+    const separator = this.pageSeparator ?? "\n---\n";
+    const textChunks = text.split(separator);
+    return textChunks.map(
+      (docChunk: string) =>
+        new Document({
+          text: docChunk,
+        }),
+    );
+  }
+
   static async getMimeType(
     data: Uint8Array,
   ): Promise<{ mime: string; extension: string }> {