Merge branch 'main' into logan/update_docs

nit: fix README instructions for v0.10 compatibility

README.md

@@ -1,6 +1,6 @@
 # LlamaParse (Preview)
 
-LlamaParse is an API created by LlamaIndex to effeciently parse and represent files for effecient retrieval and context augmentation using LlamaIndex frameworks.
+LlamaParse is an API created by LlamaIndex to efficiently parse and represent files for efficient retrieval and context augmentation using LlamaIndex frameworks.
 
 LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).
 
@@ -26,7 +26,8 @@ from llama_parse import LlamaParse
 
 parser = LlamaParse(
     api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
-    result_type="markdown"  # "markdown" and "text" are available
+    result_type="markdown",  # "markdown" and "text" are available
+    verbose=True
 )
 
 # sync
@@ -49,7 +50,8 @@ from llama_index.core import SimpleDirectoryReader
 
 parser = LlamaParse(
     api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
-    result_type="markdown"  # "markdown" and "text" are available
+    result_type="markdown",  # "markdown" and "text" are available
+    verbose=True
 )
 
 file_extractor = {".pdf": parser}
@@ -60,8 +62,12 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
 
 ## Examples
 
-Serveral end-to-end indexing examples can be found in the examples folder
+Several end-to-end indexing examples can be found in the examples folder
 
 - [Getting Started](examples/demo_basic.ipynb)
 - [Advanced RAG Example](examples/demo_advanced.ipynb)
 - [Raw API Usage](examples/demo_api.ipynb)
+
+## Terms of Service
+
+See the [Terms of Service Here](./TOS.pdf).

llama_parse/base.py

@@ -1,3 +1,4 @@
+import os
 import asyncio
 import httpx
 import mimetypes
@@ -5,9 +6,10 @@ import time
 from enum import Enum
 from typing import List, Optional
 
-from llama_index.bridge.pydantic import Field, validator
-from llama_index.readers.base import BasePydanticReader
-from llama_index.schema import Document
+from llama_index.core.bridge.pydantic import Field, validator
+from llama_index.core.constants import DEFAULT_BASE_URL
+from llama_index.core.readers.base import BasePydanticReader
+from llama_index.core.schema import Document
 
 
 class ResultType(str, Enum):
@@ -22,7 +24,7 @@ class LlamaParse(BasePydanticReader):
 
     api_key: str = Field(default="", description="The API key for the LlamaParse API.")
     base_url: str = Field(
-        default="https://api.cloud.llamaindex.ai/api/parsing",
+        default=DEFAULT_BASE_URL,
         description="The base URL of the Llama Parsing API.",
     )
     result_type: ResultType = Field(
@@ -36,6 +38,9 @@ class LlamaParse(BasePydanticReader):
         default=2000,
         description="The maximum timeout in seconds to wait for the parsing to finish.",
     )
+    verbose: bool = Field(
+        default=True, description="Whether to print the progress of the parsing."
+    )
 
     @validator("api_key", pre=True, always=True)
     def validate_api_key(cls, v: str) -> str:
@@ -48,6 +53,12 @@ class LlamaParse(BasePydanticReader):
             return api_key
         
         return v
+    
+    @validator("base_url", pre=True, always=True)
+    def validate_base_url(cls, v: str) -> str:
+        """Validate the base URL."""
+        url = os.getenv("LLAMA_CLOUD_BASE_URL", None)
+        return url or v or DEFAULT_BASE_URL
 
     def load_data(self, file_path: str, extra_info: Optional[dict] = None) -> List[Document]:
         """Load data from the input path."""
@@ -70,33 +81,44 @@ class LlamaParse(BasePydanticReader):
             files = {"file": (f.name, f, mime_type)}
 
             # send the request, start job
-            url = f"{self.base_url}/upload"
-            async with httpx.AsyncClient() as client:
+            url = f"{self.base_url}/api/parsing/upload"
+            async with httpx.AsyncClient(timeout=self.max_timeout) as client:
                 response = await client.post(url, files=files, headers=headers)
                 if not response.is_success:
                     raise Exception(f"Failed to parse the PDF file: {response.text}")
 
         # check the status of the job, return when done
         job_id = response.json()["id"]
-        print("Started parsing the file under job_id %s" % job_id)
-        result_url = f"{self.base_url}/job/{job_id}/result/{self.result_type.value}"
+        if self.verbose:
+            print("Started parsing the file under job_id %s" % job_id)
+        
+        result_url = f"{self.base_url}/api/parsing/job/{job_id}/result/{self.result_type.value}"
 
         start = time.time()
+        tries = 0
         while True:
             await asyncio.sleep(self.check_interval)
-            async with httpx.AsyncClient() as client:    
+            async with httpx.AsyncClient(timeout=self.max_timeout) as client:    
                 result = await client.get(result_url, headers=headers)
 
-                if not result.is_success:
-                    if time.time() - start > self.max_timeout:
+                if result.status_code == 404:
+                    end = time.time()
+                    if end - start > self.max_timeout:
                         raise Exception(
                             f"Timeout while parsing the PDF file: {response.text}"
                         )
+                    if self.verbose and tries % 10 == 0:
+                        print(".", end="", flush=True)
                     continue
 
+                if result.status_code == 400:
+                    detail = result.json().get("detail", "Unknown error")
+                    raise Exception(f"Failed to parse the PDF file: {detail}")
+
                 return [
                     Document(
                         text=result.json()[self.result_type.value],
                         metadata=extra_info,
                     )
                 ]
+            tries += 1

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "llama-parse"
-version = "0.1.3"
+version = "0.3.1"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"
@@ -9,8 +9,7 @@ packages = [{include = "llama_parse"}]
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
-llama-index = "^0.9.40"
-
+llama-index-core = ">=0.10.6.post1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"