vbump

Co-authored-by: Logan <logan.markewich@live.com>

.gitignore

@@ -2,3 +2,4 @@
 __pycache__/
 *.pyc
 .DS_Store
+.idea

README.md

@@ -1,15 +1,26 @@
 # LlamaParse
 
-LlamaParse is an API created by LlamaIndex to efficiently parse and represent files for efficient retrieval and context augmentation using LlamaIndex frameworks.
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/llama-parse)](https://pypi.org/project/llama-parse/)
+[![GitHub contributors](https://img.shields.io/github/contributors/run-llama/llama_parse)](https://github.com/run-llama/llama_parse/graphs/contributors)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.gg/dGcwcsnxhU)
+
+LlamaParse is a **GenAI-native document parser** that can parse complex document data for any downstream LLM use case (RAG, agents).
+
+It is really good at the following:
+
+- ✅ **Broad file type support**: Parsing a variety of unstructured file types (.pdf, .pptx, .docx, .xlsx, .html) with text, tables, visual elements, weird layouts, and more.
+- ✅ **Table recognition**: Parsing embedded tables accurately into text and semi-structured representations.
+- ✅ **Multimodal parsing and chunking**: Extracting visual elements (images/diagrams) into structured formats and return image chunks using the latest multimodal models.
+- ✅ **Custom parsing**: Input custom prompt instructions to customize the output the way you want it.
 
 LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).
 
-Free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page.
-
-There is a sandbox available to test the API [**https://cloud.llamaindex.ai/parse ↗**](https://cloud.llamaindex.ai/parse).
+The free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page by default. There is a sandbox available to test the API [**https://cloud.llamaindex.ai/parse ↗**](https://cloud.llamaindex.ai/parse).
 
 Read below for some quickstart information, or see the [full documentation](https://docs.cloud.llamaindex.ai/).
 
+If you're a company interested in enterprise RAG solutions, and/or high volume/on-prem usage of LlamaParse, come [talk to us](https://www.llamaindex.ai/contact).
+
 ## Getting Started
 
 First, login and get an api-key from [**https://cloud.llamaindex.ai/api-key ↗**](https://cloud.llamaindex.ai/api-key).
@@ -126,3 +137,9 @@ Several end-to-end indexing examples can be found in the examples folder
 ## Terms of Service
 
 See the [Terms of Service Here](./TOS.pdf).
+
+## Get in Touch (LlamaCloud)
+
+LlamaParse is part of LlamaCloud, our e2e enterprise RAG platform that provides out-of-the-box, production-ready connectors, indexing, and retrieval over your complex data sources. We offer SaaS and VPC options.
+
+LlamaCloud is currently available via waitlist (join by [creating an account](https://cloud.llamaindex.ai/)). If you're interested in state-of-the-art quality and in centralizing your RAG efforts, come [get in touch with us](https://www.llamaindex.ai/contact).

llama_parse/base.py

@@ -1,17 +1,22 @@
 import os
 import asyncio
+from io import TextIOWrapper
+
 import httpx
 import mimetypes
 import time
-from pathlib import Path
-from typing import AsyncGenerator, List, Optional, Union
+from pathlib import Path, PurePath, PurePosixPath
+from typing import AsyncGenerator, Any, Dict, List, Optional, Union
 from contextlib import asynccontextmanager
 from io import BufferedIOBase
 
+from fsspec import AbstractFileSystem
+from fsspec.spec import AbstractBufferedFile
 from llama_index.core.async_utils import run_jobs
 from llama_index.core.bridge.pydantic import Field, field_validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
+from llama_index.core.readers.file.base import get_default_fs
 from llama_index.core.schema import Document
 from llama_parse.utils import (
     nest_asyncio_err,
@@ -178,7 +183,10 @@ class LlamaParse(BasePydanticReader):
 
     # upload a document and get back a job_id
     async def _create_job(
-        self, file_input: FileInput, extra_info: Optional[dict] = None
+        self,
+        file_input: FileInput,
+        extra_info: Optional[dict] = None,
+        fs: Optional[AbstractFileSystem] = None,
     ) -> str:
         headers = {"Authorization": f"Bearer {self.api_key}"}
         url = f"{self.base_url}/api/parsing/upload"
@@ -193,7 +201,7 @@ class LlamaParse(BasePydanticReader):
             file_name = extra_info["file_name"]
             mime_type = mimetypes.guess_type(file_name)[0]
             files = {"file": (file_name, file_input, mime_type)}
-        elif isinstance(file_input, (str, Path)):
+        elif isinstance(file_input, (str, Path, PurePosixPath, PurePath)):
             file_path = str(file_input)
             file_ext = os.path.splitext(file_path)[1].lower()
             if file_ext not in SUPPORTED_FILE_TYPES:
@@ -203,7 +211,9 @@ class LlamaParse(BasePydanticReader):
                 )
             mime_type = mimetypes.guess_type(file_path)[0]
             # Open the file here for the duration of the async context
-            file_handle = open(file_path, "rb")
+            # load data, set the mime type
+            fs = fs or get_default_fs()
+            file_handle = fs.open(file_input, "rb")
             files = {"file": (os.path.basename(file_path), file_handle, mime_type)}
         else:
             raise ValueError(
@@ -259,9 +269,15 @@ class LlamaParse(BasePydanticReader):
             if file_handle is not None:
                 file_handle.close()
 
+    @staticmethod
+    def __get_filename(f: Union[TextIOWrapper, AbstractBufferedFile]) -> str:
+        if isinstance(f, TextIOWrapper):
+            return f.name
+        return f.full_name
+
     async def _get_job_result(
         self, job_id: str, result_type: str, verbose: bool = False
-    ) -> dict:
+    ) -> Dict[str, Any]:
         result_url = f"{self.base_url}/api/parsing/job/{job_id}/result/{result_type}"
         status_url = f"{self.base_url}/api/parsing/job/{job_id}"
         headers = {"Authorization": f"Bearer {self.api_key}"}
@@ -300,21 +316,16 @@ class LlamaParse(BasePydanticReader):
 
                     await asyncio.sleep(self.check_interval)
 
-                    continue
-                else:
-                    raise Exception(
-                        f"Failed to parse the file: {job_id}, status: {status}"
-                    )
-
     async def _aload_data(
         self,
         file_path: FileInput,
         extra_info: Optional[dict] = None,
+        fs: Optional[AbstractFileSystem] = None,
         verbose: bool = False,
     ) -> List[Document]:
         """Load data from the input path."""
         try:
-            job_id = await self._create_job(file_path, extra_info=extra_info)
+            job_id = await self._create_job(file_path, extra_info=extra_info, fs=fs)
             if verbose:
                 print("Started parsing the file under job_id %s" % job_id)
 
@@ -345,17 +356,19 @@ class LlamaParse(BasePydanticReader):
         self,
         file_path: Union[List[FileInput], FileInput],
         extra_info: Optional[dict] = None,
+        fs: Optional[AbstractFileSystem] = None,
     ) -> List[Document]:
         """Load data from the input path."""
         if isinstance(file_path, (str, Path, bytes, BufferedIOBase)):
             return await self._aload_data(
-                file_path, extra_info=extra_info, verbose=self.verbose
+                file_path, extra_info=extra_info, fs=fs, verbose=self.verbose
             )
         elif isinstance(file_path, list):
             jobs = [
                 self._aload_data(
                     f,
                     extra_info=extra_info,
+                    fs=fs,
                     verbose=self.verbose and not self.show_progress,
                 )
                 for f in file_path
@@ -384,10 +397,11 @@ class LlamaParse(BasePydanticReader):
         self,
         file_path: Union[List[FileInput], FileInput],
         extra_info: Optional[dict] = None,
+        fs: Optional[AbstractFileSystem] = None,
     ) -> List[Document]:
         """Load data from the input path."""
         try:
-            return asyncio.run(self.aload_data(file_path, extra_info))
+            return asyncio.run(self.aload_data(file_path, extra_info, fs=fs))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
@@ -402,12 +416,13 @@ class LlamaParse(BasePydanticReader):
             job_id = await self._create_job(file_path, extra_info=extra_info)
             if self.verbose:
                 print("Started parsing the file under job_id %s" % job_id)
-
             result = await self._get_job_result(job_id, "json")
             result["job_id"] = job_id
-            result["file_path"] = file_path
-            return [result]
 
+            if not isinstance(file_path, (bytes, BufferedIOBase)):
+                result["file_path"] = str(file_path)
+
+            return [result]
         except Exception as e:
             file_repr = file_path if isinstance(file_path, str) else "<bytes/buffer>"
             print(f"Error while parsing the file '{file_repr}':", e)
@@ -492,7 +507,9 @@ class LlamaParse(BasePydanticReader):
 
                         image["path"] = image_path
                         image["job_id"] = job_id
-                        image["original_pdf_path"] = result["file_path"]
+
+                        image["original_file_path"] = result.get("file_path", None)
+
                         image["page_number"] = page["page"]
                         with open(image_path, "wb") as f:
                             image_url = f"{self.base_url}/api/parsing/job/{job_id}/result/image/{image_name}"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "llama-parse"
-version = "0.5.3"
+version = "0.5.5"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"

tests/test_reader.py

@@ -1,6 +1,8 @@
 import os
 import pytest
+from fsspec.implementations.local import LocalFileSystem
 from httpx import AsyncClient
+
 from llama_parse import LlamaParse
 
 
@@ -70,6 +72,20 @@ def test_simple_page_markdown_buffer(markdown_parser: LlamaParse) -> None:
         assert len(result[0].text) > 0
 
 
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_with_custom_fs() -> None:
+    parser = LlamaParse(result_type="markdown")
+    fs = LocalFileSystem()
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
+    result = parser.load_data(filepath, fs=fs)
+    assert len(result) == 1
+
+
 @pytest.mark.skipif(
     os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
     reason="LLAMA_CLOUD_API_KEY not set",