cr

* Update issue templates

* Update issue templates

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,8 +7,6 @@ assignees: ''
 
 ---
 
-_Note: we're aware of some missing content in the output and layout issues on tables. Please refrain from opening new issues on this topic unless if you think it's different from what has already been reported._
-
 **Describe the bug**
 Write a concise description of what the bug is.
 
@@ -19,19 +17,15 @@ If possible, please provide the PDF file causing the issue.
 If you have it, please provide the ID of the job you ran.
 You can find it here: https://cloud.llamaindex.ai/parse in the "History" tab.
 
-**Screenshots**
-Feel free to also provide screenshots if relevant.
-
 **Client:**
 Please remove untested options:
- - Frontend (cloud.llamaindex.ai)
  - Python Library
+ - API
+ - Frontend (cloud.llamaindex.ai)
  - Typescript Library
  - Notebook
- - API
-
-**Options**
-What options did you use? Multimodal, fast mode, parsing instructions, etc.
 
 **Additional context**
 Add any additional context about the problem here.
+What options did you use? Premium mode, multimodal, fast mode, parsing instructions, etc.
+Screenshots, code snippets, etc.

README.md

@@ -38,7 +38,22 @@ Lastly, install the package:
 
 `pip install llama-parse`
 
-Now you can run the following to parse your first PDF file:
+Now you can parse your first PDF file using the command line interface. Use the command `llama-parse [file_paths]`. See the help text with `llama-parse --help`.
+
+```bash
+export LLAMA_CLOUD_API_KEY='llx-...'
+
+# output as text
+llama-parse my_file.pdf --result-type text --output-file output.txt
+
+# output as markdown
+llama-parse my_file.pdf --result-type markdown --output-file output.md
+
+# output as raw json
+llama-parse my_file.pdf --output-raw-json --output-file output.json
+```
+
+You can also create simple scripts:
 
 ```python
 import nest_asyncio
@@ -87,13 +102,18 @@ parser = LlamaParse(
     language="en",  # Optionally you can define a language, default=en
 )
 
-with open("./my_file1.pdf", "rb") as f:
-    documents = parser.load_data(f)
+file_name = "my_file1.pdf"
+extra_info = {"file_name": file_name}
+
+with open(f"./{file_name}", "rb") as f:
+    # must provide extra_info with file_name key with passing file object
+    documents = parser.load_data(f, extra_info=extra_info)
 
 # you can also pass file bytes directly
-with open("./my_file1.pdf", "rb") as f:
+with open(f"./{file_name}", "rb") as f:
     file_bytes = f.read()
-    documents = parser.load_data(file_bytes)
+    # must provide extra_info with file_name key with passing file bytes
+    documents = parser.load_data(file_bytes, extra_info=extra_info)
 ```
 
 ## Using with `SimpleDirectoryReader`

examples/demo_json.ipynb

@@ -342,7 +342,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "llama-parse-aNC435Vv-py3.10",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },

examples/test_tesla_impact_report/test_gpt4o.ipynb

@@ -46,7 +46,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"<LLAMA_CLOUD_API_KEY>"
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"<LLAMA_CLOUD_API_KEY>\""
    ]
   },
   {

llama_parse/base.py

@@ -12,7 +12,7 @@ from io import BufferedIOBase
 
 from fsspec import AbstractFileSystem
 from fsspec.spec import AbstractBufferedFile
-from llama_index.core.async_utils import run_jobs
+from llama_index.core.async_utils import asyncio_run, run_jobs
 from llama_index.core.bridge.pydantic import Field, field_validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
@@ -95,6 +95,10 @@ class LlamaParse(BasePydanticReader):
         default=False,
         description="Use our best parser mode if set to True.",
     )
+    continuous_mode: bool = Field(
+        default=False,
+        description="Parse documents continuously, leading to better results on documents where tables span across two pages.",
+    )
     do_not_unroll_columns: Optional[bool] = Field(
         default=False,
         description="If set to true, the parser will keep column in the text according to document layout. Reduce reconstruction accuracy, and LLM's/embedings performances in most case.",
@@ -119,6 +123,10 @@ class LlamaParse(BasePydanticReader):
         default=None,
         description="The API key for the GPT-4o API. Lowers the cost of parsing.",
     )
+    guess_xlsx_sheet_names: Optional[bool] = Field(
+        default=False,
+        description="Whether to guess the sheet names of the xlsx file.",
+    )
     bounding_box: Optional[str] = Field(
         default=None,
         description="The bounding box to use to extract text from documents describe as a string containing the bounding box margins",
@@ -154,6 +162,34 @@ class LlamaParse(BasePydanticReader):
     custom_client: Optional[httpx.AsyncClient] = Field(
         default=None, description="A custom HTTPX client to use for sending requests."
     )
+    disable_ocr: bool = Field(
+        default=False,
+        description="Disable the OCR on the document. LlamaParse will only extract the copyable text from the document.",
+    )
+    is_formatting_instruction: bool = Field(
+        default=True,
+        description="Allow the parsing instruction to also format the output. Disable to have a cleaner markdown output.",
+    )
+    annotate_links: bool = Field(
+        default=False,
+        description="Annotate links found in the document to extract their URL.",
+    )
+    webhook_url: Optional[str] = Field(
+        default=None,
+        description="A URL that needs to be called at the end of the parsing job.",
+    )
+    azure_openai_deployment_name: Optional[str] = Field(
+        default=None, description="Azure Openai Deployment Name"
+    )
+    azure_openai_endpoint: Optional[str] = Field(
+        default=None, description="Azure Openai Endpoint"
+    )
+    azure_openai_api_version: Optional[str] = Field(
+        default=None, description="Azure Openai API Version"
+    )
+    azure_openai_key: Optional[str] = Field(
+        default=None, description="Azure Openai Key"
+    )
 
     @field_validator("api_key", mode="before", check_fields=True)
     @classmethod
@@ -232,6 +268,7 @@ class LlamaParse(BasePydanticReader):
             "do_not_cache": self.do_not_cache,
             "fast_mode": self.fast_mode,
             "premium_mode": self.premium_mode,
+            "continuous_mode": self.continuous_mode,
             "do_not_unroll_columns": self.do_not_unroll_columns,
             "gpt4o_mode": self.gpt4o_mode,
             "gpt4o_api_key": self.gpt4o_api_key,
@@ -239,6 +276,11 @@ class LlamaParse(BasePydanticReader):
             "use_vendor_multimodal_model": self.use_vendor_multimodal_model,
             "vendor_multimodal_model_name": self.vendor_multimodal_model_name,
             "take_screenshot": self.take_screenshot,
+            "disable_ocr": self.disable_ocr,
+            "guess_xlsx_sheet_names": self.guess_xlsx_sheet_names,
+            "is_formatting_instruction": self.is_formatting_instruction,
+            "annotate_links": self.annotate_links,
+            "from_python_package": True,
         }
 
         # only send page separator to server if it is not None
@@ -258,6 +300,22 @@ class LlamaParse(BasePydanticReader):
         if self.target_pages is not None:
             data["target_pages"] = self.target_pages
 
+        if self.webhook_url is not None:
+            data["webhook_url"] = self.webhook_url
+
+        # Azure OpenAI
+        if self.azure_openai_deployment_name is not None:
+            data["azure_openai_deployment_name"] = self.azure_openai_deployment_name
+
+        if self.azure_openai_endpoint is not None:
+            data["azure_openai_endpoint"] = self.azure_openai_endpoint
+
+        if self.azure_openai_api_version is not None:
+            data["azure_openai_api_version"] = self.azure_openai_api_version
+
+        if self.azure_openai_key is not None:
+            data["azure_openai_key"] = self.azure_openai_key
+
         try:
             async with self.client_context() as client:
                 response = await client.post(
@@ -308,7 +366,8 @@ class LlamaParse(BasePydanticReader):
                     continue
 
                 # Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
-                status = result.json()["status"]
+                result_json = result.json()
+                status = result_json["status"]
                 if status == "SUCCESS":
                     parsed_result = await client.get(result_url, headers=headers)
                     return parsed_result.json()
@@ -320,6 +379,14 @@ class LlamaParse(BasePydanticReader):
                         print(".", end="", flush=True)
 
                     await asyncio.sleep(self.check_interval)
+                else:
+                    error_code = result_json.get("error_code", "No error code found")
+                    error_message = result_json.get(
+                        "error_message", "No error message found"
+                    )
+
+                    exception_str = f"Job ID: {job_id} failed with status: {status}, Error code: {error_code}, Error message: {error_message}"
+                    raise Exception(exception_str)
 
     async def _aload_data(
         self,
@@ -364,7 +431,7 @@ class LlamaParse(BasePydanticReader):
         fs: Optional[AbstractFileSystem] = None,
     ) -> List[Document]:
         """Load data from the input path."""
-        if isinstance(file_path, (str, Path, bytes, BufferedIOBase)):
+        if isinstance(file_path, (str, PurePosixPath, Path, bytes, BufferedIOBase)):
             return await self._aload_data(
                 file_path, extra_info=extra_info, fs=fs, verbose=self.verbose
             )
@@ -406,7 +473,7 @@ class LlamaParse(BasePydanticReader):
     ) -> List[Document]:
         """Load data from the input path."""
         try:
-            return asyncio.run(self.aload_data(file_path, extra_info, fs=fs))
+            return asyncio_run(self.aload_data(file_path, extra_info, fs=fs))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
@@ -473,7 +540,7 @@ class LlamaParse(BasePydanticReader):
     ) -> List[dict]:
         """Parse the input path."""
         try:
-            return asyncio.run(self.aget_json(file_path, extra_info))
+            return asyncio_run(self.aget_json(file_path, extra_info))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
@@ -536,7 +603,61 @@ class LlamaParse(BasePydanticReader):
     def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
         """Download images from the parsed result."""
         try:
-            return asyncio.run(self.aget_images(json_result, download_path))
+            return asyncio_run(self.aget_images(json_result, download_path))
+        except RuntimeError as e:
+            if nest_asyncio_err in str(e):
+                raise RuntimeError(nest_asyncio_msg)
+            else:
+                raise e
+
+    async def aget_xlsx(
+        self, json_result: List[dict], download_path: str
+    ) -> List[dict]:
+        """Download images from the parsed result."""
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+
+        # make the download path
+        if not os.path.exists(download_path):
+            os.makedirs(download_path)
+        try:
+            xlsx_list = []
+            for result in json_result:
+                job_id = result["job_id"]
+                if self.verbose:
+                    print("> XLSX")
+
+                xlsx_path = os.path.join(download_path, f"{job_id}.xlsx")
+
+                xlsx = {}
+
+                xlsx["path"] = xlsx_path
+                xlsx["job_id"] = job_id
+                xlsx["original_file_path"] = result.get("file_path", None)
+
+                with open(xlsx_path, "wb") as f:
+                    xlsx_url = (
+                        f"{self.base_url}/api/parsing/job/{job_id}/result/raw/xlsx"
+                    )
+                    async with self.client_context() as client:
+                        res = await client.get(
+                            xlsx_url, headers=headers, timeout=self.max_timeout
+                        )
+                        res.raise_for_status()
+                        f.write(res.content)
+                xlsx_list.append(xlsx)
+            return xlsx_list
+
+        except Exception as e:
+            print("Error while downloading xlsx:", e)
+            if self.ignore_errors:
+                return []
+            else:
+                raise e
+
+    def get_xlsx(self, json_result: List[dict], download_path: str) -> List[dict]:
+        """Download xlsx from the parsed result."""
+        try:
+            return asyncio_run(self.aget_xlsx(json_result, download_path))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)

llama_parse/cli/main.py

@@ -0,0 +1,92 @@
+import click
+import json
+from enum import Enum
+from pathlib import Path
+from pydantic.fields import FieldInfo
+from typing import Any, Callable, List
+
+from llama_parse.base import LlamaParse
+
+
+def pydantic_field_to_click_option(name: str, field: FieldInfo) -> click.Option:
+    """Convert a Pydantic field to a Click option."""
+    kwargs = {
+        "default": field.default if field.default else None,
+        "help": field.description,
+    }
+
+    if isinstance(kwargs["default"], Enum):
+        kwargs["default"] = kwargs["default"].value
+
+    if field.annotation is bool:
+        kwargs["is_flag"] = True
+        if field.default and field.default is True:
+            name = f"no-{name}"
+    return click.option(f'--{name.replace("_", "-")}', **kwargs)
+
+
+def add_options(options: List[click.Option]) -> Callable:
+    def _add_options(func: Callable) -> Callable:
+        for option in reversed(options):
+            func = option(func)
+        return func
+
+    return _add_options
+
+
+@click.command()
+@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--output-file", type=click.Path(path_type=Path), help="Path to save the output"
+)
+@click.option("--output-raw-json", is_flag=True, help="Output the raw JSON result")
+@add_options(
+    [
+        pydantic_field_to_click_option(name, field)
+        for name, field in LlamaParse.model_fields.items()
+        if name not in ["custom_client"]
+    ]
+)
+def parse(**kwargs: Any) -> None:
+    """Parse files using LlamaParse and output the results."""
+    file_paths = kwargs.pop("file_paths")
+    output_file = kwargs.pop("output_file")
+    output_raw_json = kwargs.pop("output_raw_json")
+
+    # Remove None values to use LlamaParse defaults
+    kwargs = {k: v for k, v in kwargs.items() if v is not None}
+
+    # Remove no- prefix for boolean flags
+    kwargs = {k.replace("no_", ""): v for k, v in kwargs.items()}
+
+    parser = LlamaParse(**kwargs)
+    if output_raw_json:
+        results = parser.get_json_result(list(file_paths))
+
+        if output_file:
+            with output_file.open("w") as f:
+                json.dump(results, f)
+            click.echo(f"Results saved to {output_file}")
+        else:
+            click.echo(results)
+    else:
+        results = parser.load_data(list(file_paths))
+
+        if output_file:
+            with output_file.open("w") as f:
+                for i, doc in enumerate(results):
+                    f.write(f"File: {doc.metadata.get('file_path', 'Unknown')}\n")  # type: ignore
+                    f.write(doc.text)  # type: ignore
+                    if i < len(results) - 1:
+                        f.write("\n\n---\n\n")
+            click.echo(f"Results saved to {output_file}")
+        else:
+            for i, doc in enumerate(results):
+                click.echo(f"File: {doc.metadata.get('file_path', 'Unknown')}")  # type: ignore
+                click.echo(doc.text)  # type: ignore
+                if i < len(results) - 1:
+                    click.echo("\n---\n")
+
+
+if __name__ == "__main__":
+    parse()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "llama-parse"
-version = "0.5.6"
+version = "0.5.13"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"
@@ -14,7 +14,11 @@ packages = [{include = "llama_parse"}]
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
 llama-index-core = ">=0.11.0"
+click = "^8.1.7"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
 ipykernel = "^6.29.0"
+
+[tool.poetry.scripts]
+llama-parse = "llama_parse.cli.main:parse"