vbump

Co-authored-by: Logan <logan.markewich@live.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,8 @@ assignees: ''
 
 ---
 
+_Note: we're aware of some missing content in the output and layout issues on tables. Please refrain from opening new issues on this topic unless if you think it's different from what has already been reported._
+
 **Describe the bug**
 Write a concise description of what the bug is.
 
@@ -17,15 +19,19 @@ If possible, please provide the PDF file causing the issue.
 If you have it, please provide the ID of the job you ran.
 You can find it here: https://cloud.llamaindex.ai/parse in the "History" tab.
 
+**Screenshots**
+Feel free to also provide screenshots if relevant.
+
 **Client:**
 Please remove untested options:
- - Python Library
- - API
  - Frontend (cloud.llamaindex.ai)
+ - Python Library
  - Typescript Library
  - Notebook
+ - API
+
+**Options**
+What options did you use? Multimodal, fast mode, parsing instructions, etc.
 
 **Additional context**
 Add any additional context about the problem here.
-What options did you use? Premium mode, multimodal, fast mode, parsing instructions, etc.
-Screenshots, code snippets, etc.

README.md

@@ -38,22 +38,7 @@ Lastly, install the package:
 
 `pip install llama-parse`
 
-Now you can parse your first PDF file using the command line interface. Use the command `llama-parse [file_paths]`. See the help text with `llama-parse --help`.
-
-```bash
-export LLAMA_CLOUD_API_KEY='llx-...'
-
-# output as text
-llama-parse my_file.pdf --result-type text --output-file output.txt
-
-# output as markdown
-llama-parse my_file.pdf --result-type markdown --output-file output.md
-
-# output as raw json
-llama-parse my_file.pdf --output-raw-json --output-file output.json
-```
-
-You can also create simple scripts:
+Now you can run the following to parse your first PDF file:
 
 ```python
 import nest_asyncio
@@ -102,18 +87,13 @@ parser = LlamaParse(
     language="en",  # Optionally you can define a language, default=en
 )
 
-file_name = "my_file1.pdf"
-extra_info = {"file_name": file_name}
-
-with open(f"./{file_name}", "rb") as f:
-    # must provide extra_info with file_name key with passing file object
-    documents = parser.load_data(f, extra_info=extra_info)
+with open("./my_file1.pdf", "rb") as f:
+    documents = parser.load_data(f)
 
 # you can also pass file bytes directly
-with open(f"./{file_name}", "rb") as f:
+with open("./my_file1.pdf", "rb") as f:
     file_bytes = f.read()
-    # must provide extra_info with file_name key with passing file bytes
-    documents = parser.load_data(file_bytes, extra_info=extra_info)
+    documents = parser.load_data(file_bytes)
 ```
 
 ## Using with `SimpleDirectoryReader`

examples/demo_json.ipynb

@@ -342,7 +342,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "llama-parse-aNC435Vv-py3.10",
    "language": "python",
    "name": "python3"
   },

examples/multimodal/multimodal_rag_slide_deck.ipynb

@@ -165,18 +165,7 @@
    "execution_count": null,
    "id": "ef82a985-4088-4bb7-9a21-0318e1b9207d",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Parsing text...\n",
-      "Started parsing the file under job_id 62f157a9-9ef9-4e5b-95ac-67093fa25800\n",
-      "..........Parsing PDF file...\n",
-      "Started parsing the file under job_id 1ddd5654-062b-4e19-b488-d66efc9c509d\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print(f\"Parsing text...\")\n",
     "docs_text = parser_text.load_data(\"data/conocophillips.pdf\")\n",
@@ -185,36 +174,42 @@
     "md_json_list = md_json_objs[0][\"pages\"]"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7506b603-c01f-45de-b354-4a0728dde03c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(docs_text[0].get_content())"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "5318fb7b-fe6a-4a8a-b82e-4ed7b4512c37",
    "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(md_json_list[10][\"md\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7a46a73e-c6e2-4b0b-bd10-31b0d3e4b70f",
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "# Commitment to Disciplined Reinvestment Rate\n",
-      "\n",
-      "| Period       | Description                          | Reinvestment Rate | WTI Average |\n",
-      "|--------------|--------------------------------------|-------------------|-------------|\n",
-      "| 2012-2016    | Industry Growth Focus                | >100%             | ~$75/BBL    |\n",
-      "| 2017-2022    | ConocoPhillips Strategy Reset        | <60%              | ~$63/BBL    |\n",
-      "| 2023E        |                                      |                   | at $80/BBL  |\n",
-      "| 2024-2028    | Disciplined Reinvestment Rate        | ~50%              | at $60/BBL  |\n",
-      "| 2029-2032    |                                      | ~6% CFO CAGR      | at $60/BBL  |\n",
-      "\n",
-      "- **Historic Reinvestment Rate**: Gray bars\n",
-      "- **Reinvestment Rate at $60/BBL WTI**: Blue bars\n",
-      "- **Reinvestment Rate at $80/BBL WTI**: Dashed blue lines\n",
-      "\n",
-      "Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n"
+      "dict_keys(['page', 'text', 'md', 'images', 'items'])\n"
      ]
     }
    ],
    "source": [
-    "print(md_json_list[10][\"md\"])"
+    "print(md_json_list[1].keys())"
    ]
   },
   {
@@ -304,7 +299,7 @@
     "    image_files = _get_sorted_image_files(image_dir) if image_dir is not None else None\n",
     "    md_texts = [d[\"md\"] for d in json_dicts] if json_dicts is not None else None\n",
     "\n",
-    "    doc_chunks = [c for d in docs for c in d.text.split(\"---\")]\n",
+    "    doc_chunks = docs[0].text.split(\"---\")\n",
     "    for idx, doc_chunk in enumerate(doc_chunks):\n",
     "        chunk_metadata = {\"page_num\": idx + 1}\n",
     "        if image_files is not None:\n",
@@ -344,23 +339,25 @@
      "output_type": "stream",
      "text": [
       "page_num: 11\n",
-      "image_path: data_images/1ddd5654-062b-4e19-b488-d66efc9c509d-page_39.jpg\n",
+      "image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-10.jpg\n",
       "parsed_text_markdown: # Commitment to Disciplined Reinvestment Rate\n",
       "\n",
-      "| Period       | Description                          | Reinvestment Rate | WTI Average |\n",
-      "|--------------|--------------------------------------|-------------------|-------------|\n",
-      "| 2012-2016    | Industry Growth Focus                | >100%             | ~$75/BBL    |\n",
-      "| 2017-2022    | ConocoPhillips Strategy Reset        | <60%              | ~$63/BBL    |\n",
-      "| 2023E        |                                      |                   | at $80/BBL  |\n",
-      "| 2024-2028    | Disciplined Reinvestment Rate        | ~50%              | at $60/BBL  |\n",
-      "| 2029-2032    |                                      | ~6% CFO CAGR      | at $60/BBL  |\n",
+      "| Year       | Reinvestment Rate | WTI Average Price | Reinvestment Rate at $60/BBL WTI | Reinvestment Rate at $80/BBL WTI |\n",
+      "|------------|-------------------|-------------------|----------------------------------|----------------------------------|\n",
+      "| 2012-2016  | >100%             | ~$75/BBL          |                                  |                                  |\n",
+      "| 2017-2022  | <60%              | ~$63/BBL          |                                  |                                  |\n",
+      "| 2023E      |                   |                   |                                  | at $80/BBL WTI                   |\n",
+      "| 2024-2028  |                   |                   | at $60/BBL WTI                   | at $80/BBL WTI                   |\n",
+      "| 2029-2032  |                   |                   | at $60/BBL WTI                   | at $80/BBL WTI                   |\n",
       "\n",
-      "- **Historic Reinvestment Rate**: Gray bars\n",
-      "- **Reinvestment Rate at $60/BBL WTI**: Blue bars\n",
-      "- **Reinvestment Rate at $80/BBL WTI**: Dashed blue lines\n",
+      "**Disciplined Reinvestment Rate is the Foundation for Superior Returns on and of Capital, while Driving Durable CFO Growth**\n",
       "\n",
-      "Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n",
-      "parsed_text: Commitment to Disciplined Reinvestment Rate\n",
+      "- ~50% 10-Year Reinvestment Rate\n",
+      "- ~6% CFO CAGR 2024-2032 at $60/BBL WTI Mid-Cycle Planning Price\n",
+      "\n",
+      "**Note:** Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n",
+      "parsed_text: \n",
+      "Commitment to Disciplined Reinvestment Rate\n",
       "                         Industry                    ConocoPhillips\n",
       "                                                     Strategy Reset                   Disciplined Reinvestment Rate is the Foundation for Superior\n",
       "                      Growth Focus                                                    Returns on and of Capital, while Driving Durable CFO Growth\n",
@@ -377,7 +374,7 @@
       "          0%\n",
       "                          2012-2016                        2017-2022                         2023E                          2024-2028                       2029-2032\n",
       "                                    Historic Reinvestment Rate            Reinvestment Rate at $60/BBL WTI                 Reinvestment Rate at $80/BBL WTI\n",
-      " Reinvestment rate and cash from operations (CFO) are non-GAAP measures: Definitions and reconciliations are included in the Appendix                                  ConocoPhillips\n"
+      " Reinvestment rate andcashfrom operations (CFO) are non-GAAP measures: Definitions and reconciliations are included in the Appendix                                    ConocoPhillips\n"
      ]
     }
    ],
@@ -400,17 +397,7 @@
    "execution_count": null,
    "id": "6ea53c31-0e38-421c-8d9b-0e3adaa1677e",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/jerryliu/Programming/gpt_index/.venv/lib/python3.10/site-packages/tiktoken/core.py:50: RuntimeWarning: coroutine 'LlamaParse.aload_data' was never awaited\n",
-      "  self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str)\n",
-      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "from llama_index.core import (\n",
@@ -601,7 +588,7 @@
       "                            Under $40/BBL Cost of Supply                              10-Year Plan Cumulative Production (BBOE)\n",
       "      S50                   S32/BBL                                                Lower 48                           Alaska\n",
       "                    Average Cost of Supply\n",
-      "  3 $40                                                                                                                       GKA        GWA\n",
+      "  3$40                                                                                                                        GKA        GWA\n",
       "                                                                                                                      GPA     WNS\n",
       "      $30                                                                                                             EMENA\n",
       "  3                                                                                                                              Norway\n",
@@ -612,7 +599,7 @@
       "                                                                                                                      APLNG        Montney\n",
       "       S0\n",
       "                                               10              15              20                         Bakken\n",
-      "                                         Resource (BBOE)                           Eagle Ford             Other       Malaysia ChinaSurmont\n",
+      "                                         Resource (BBOE)                           Eagle Ford             Other       MalaysiaChina Surmont\n",
       "                 Lower 48      Canada       Alaska      EMENA    Asia Pacific\n",
       "Costs assumemid-cycle price environment of S60/BBL WTI:\n",
       "                                                                                                                                           ConocoPhillips\n"
@@ -700,126 +687,70 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "d78e53cf-35cb-4ef8-b03e-1b47ba15ae64",
+   "id": "1cdce5d8-6bb3-4cd3-929d-1cec249d9052",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Added user message to memory: Tell me about the diverse geographies where Conoco Phillips has a production base\n",
+      "Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
       "=== Calling Function ===\n",
-      "Calling function: vector_tool with args: {\"input\": \"Conoco Phillips production base geographies\"}\n",
+      "Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
       "=== Function Output ===\n",
-      "ConocoPhillips' production base geographies include:\n",
+      "The ConocoPhillips capex/EUR in the Delaware Basin is $10/BOE.\n",
       "\n",
-      "1. **Lower 48** (Permian, Eagle Ford, Bakken, Other)\n",
-      "2. **Alaska** (GKA, GWA, GPA, WNS)\n",
-      "3. **EMENA** (Norway, Libya, Qatar)\n",
-      "4. **Asia Pacific** (APLNG, Malaysia, China)\n",
-      "5. **Canada** (Montney, Surmont)\n",
-      "\n",
-      "This information was derived from the image on page 14, which provides a detailed breakdown of the diverse production base and the regions involved. The parsed markdown and raw text also support this information, but the image provides the clearest and most comprehensive view. There are no discrepancies between the image and the parsed text in this case.\n",
-      "=== LLM Response ===\n",
-      "ConocoPhillips has a diverse production base spread across various geographies, including:\n",
-      "\n",
-      "1. **Lower 48**:\n",
-      "   - Permian Basin\n",
-      "   - Eagle Ford\n",
-      "   - Bakken\n",
-      "   - Other regions within the continental United States\n",
-      "\n",
-      "2. **Alaska**:\n",
-      "   - Greater Kuparuk Area (GKA)\n",
-      "   - Greater Prudhoe Area (GPA)\n",
-      "   - Greater Willow Area (GWA)\n",
-      "   - Western North Slope (WNS)\n",
-      "\n",
-      "3. **EMENA (Europe, Middle East, and North Africa)**:\n",
-      "   - Norway\n",
-      "   - Libya\n",
-      "   - Qatar\n",
-      "\n",
-      "4. **Asia Pacific**:\n",
-      "   - Australia Pacific LNG (APLNG)\n",
-      "   - Malaysia\n",
-      "   - China\n",
-      "\n",
-      "5. **Canada**:\n",
-      "   - Montney\n",
-      "   - Surmont\n",
-      "\n",
-      "These regions highlight the global reach and diverse geographical footprint of ConocoPhillips' production operations.\n",
-      "Added user message to memory: Tell me about the diverse geographies where Conoco Phillips has a production base\n",
+      "I obtained this information from the image provided. The image clearly shows a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\" where ConocoPhillips is listed with a capex/EUR of $10/BOE. This information is consistent with the parsed markdown text, which also lists ConocoPhillips' capex/EUR as $10/BOE in the Delaware Basin. There are no discrepancies between the image and the parsed markdown text in this case.\n",
       "=== Calling Function ===\n",
-      "Calling function: vector_tool with args: {\"input\": \"diverse geographies where Conoco Phillips has a production base\"}\n",
+      "Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
       "=== Function Output ===\n",
-      "ConocoPhillips has a diverse production base that includes the Lower 48 (Permian, Bakken, Eagle Ford), Alaska, Canada (Montney, Surmont), EMENA (Norway, Libya), Asia Pacific (Malaysia, China, APLNG), and Qatar.\n",
+      "The competitors' Capex/EUR in the Delaware Basin can be found in the image on the slide titled \"Delaware: Vast Inventory with Proven Track Record of Performance.\" The relevant information is presented in a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\".\n",
+      "\n",
+      "Here are the details:\n",
+      "\n",
+      "- ConocoPhillips: $10/BOE\n",
+      "- Competitor 1: $15/BOE\n",
+      "- Competitor 2: $20/BOE\n",
+      "- Competitor 3: $25/BOE\n",
+      "- Competitor 4: $30/BOE\n",
+      "- Competitor 5: $35/BOE\n",
+      "- Competitor 6: $40/BOE\n",
+      "- Competitor 7: $45/BOE\n",
+      "\n",
+      "This information was obtained directly from the image, which provides a clear visual representation of the Capex/EUR values for ConocoPhillips and its competitors in the Delaware Basin. The parsed markdown text also confirms these values, ensuring consistency between the image and the text.\n",
       "=== LLM Response ===\n",
-      "ConocoPhillips has a diverse production base spanning several key geographies:\n",
+      "The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
       "\n",
-      "1. **Lower 48 (United States)**: This includes major production areas such as the Permian Basin, Bakken Formation, and Eagle Ford Shale.\n",
-      "2. **Alaska**: Significant operations in the North Slope region.\n",
-      "3. **Canada**: Operations in the Montney Formation and the Surmont oil sands project.\n",
-      "4. **EMENA (Europe, Middle East, and North Africa)**: Notable operations in Norway and Libya.\n",
-      "5. **Asia Pacific**: Includes operations in Malaysia, China, and the Australia Pacific LNG (APLNG) project.\n",
-      "6. **Qatar**: Involvement in the country's energy sector.\n",
+      "- **ConocoPhillips**: $10/BOE\n",
+      "- **Competitor 1**: $15/BOE\n",
+      "- **Competitor 2**: $20/BOE\n",
+      "- **Competitor 3**: $25/BOE\n",
+      "- **Competitor 4**: $30/BOE\n",
+      "- **Competitor 5**: $35/BOE\n",
+      "- **Competitor 6**: $40/BOE\n",
+      "- **Competitor 7**: $45/BOE\n",
       "\n",
-      "These regions highlight the company's extensive and varied geographical footprint in the energy production industry.\n"
+      "This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n",
+      "The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
+      "\n",
+      "- **ConocoPhillips**: $10/BOE\n",
+      "- **Competitor 1**: $15/BOE\n",
+      "- **Competitor 2**: $20/BOE\n",
+      "- **Competitor 3**: $25/BOE\n",
+      "- **Competitor 4**: $30/BOE\n",
+      "- **Competitor 5**: $35/BOE\n",
+      "- **Competitor 6**: $40/BOE\n",
+      "- **Competitor 7**: $45/BOE\n",
+      "\n",
+      "This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n"
      ]
     }
    ],
    "source": [
-    "query = (\n",
-    "    \"Tell me about the diverse geographies where Conoco Phillips has a production base\"\n",
+    "# response = agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
+    "response = agent.query(\n",
+    "    \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
     ")\n",
-    "response = agent.query(query)\n",
-    "base_response = base_agent.query(query)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "355d2aa4-c26f-480e-b512-4446acbd9227",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "ConocoPhillips has a diverse production base spread across various geographies, including:\n",
-      "\n",
-      "1. **Lower 48**:\n",
-      "   - Permian Basin\n",
-      "   - Eagle Ford\n",
-      "   - Bakken\n",
-      "   - Other regions within the continental United States\n",
-      "\n",
-      "2. **Alaska**:\n",
-      "   - Greater Kuparuk Area (GKA)\n",
-      "   - Greater Prudhoe Area (GPA)\n",
-      "   - Greater Willow Area (GWA)\n",
-      "   - Western North Slope (WNS)\n",
-      "\n",
-      "3. **EMENA (Europe, Middle East, and North Africa)**:\n",
-      "   - Norway\n",
-      "   - Libya\n",
-      "   - Qatar\n",
-      "\n",
-      "4. **Asia Pacific**:\n",
-      "   - Australia Pacific LNG (APLNG)\n",
-      "   - Malaysia\n",
-      "   - China\n",
-      "\n",
-      "5. **Canada**:\n",
-      "   - Montney\n",
-      "   - Surmont\n",
-      "\n",
-      "These regions highlight the global reach and diverse geographical footprint of ConocoPhillips' production operations.\n"
-     ]
-    }
-   ],
-   "source": [
     "print(str(response))"
    ]
   },
@@ -833,82 +764,85 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "page_num: 14\n",
-      "image_path: data_images/1ddd5654-062b-4e19-b488-d66efc9c509d-page_12.jpg\n",
-      "parsed_text_markdown: # Our Differentiated Portfolio: Deep, Durable and Diverse\n",
+      "page_num: 38\n",
+      "image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-37.jpg\n",
+      "parsed_text_markdown: # Delaware: Vast Inventory with Proven Track Record of Performance\n",
       "\n",
-      "## ~20 BBOE of Resource\n",
-      "Under $40/BBL Cost of Supply\n",
+      "## Prolific Acreage Spanning Over ~659,000 Net Acres¹\n",
       "\n",
-      "### ~ $32/BBL\n",
-      "Average Cost of Supply\n",
+      "![Map of Delaware Basin](image)\n",
       "\n",
-      "### WTI Cost of Supply ($/BBL)\n",
+      "### Total 10-Year Operated Permian Inventory\n",
       "\n",
-      "| Cost ($/BBL) | Resource (BBOE) |\n",
-      "|--------------|-----------------|\n",
-      "| $0           | 0               |\n",
-      "| $10          |                 |\n",
-      "| $20          |                 |\n",
-      "| $30          |                 |\n",
-      "| $40          |                 |\n",
-      "| $50          |                 |\n",
+      "- Delaware Basin: 65%\n",
+      "- Midland Basin: 35%\n",
       "\n",
-      "- **Legend:**\n",
-      "  - Lower 48\n",
-      "  - Canada\n",
-      "  - Alaska\n",
-      "  - EMENA\n",
-      "  - Asia Pacific\n",
+      "### High Single-Digit Production Growth\n",
       "\n",
-      "*Costs assume a mid-cycle price environment of $60/BBL WTI.*\n",
+      "## 12-Month Cumulative Production³ (BOE/FT)\n",
       "\n",
-      "## Diverse Production Base\n",
-      "10-Year Plan Cumulative Production (BBOE)\n",
+      "| Months | 2019 | 2020 | 2021 | 2022 |\n",
+      "|--------|------|------|------|------|\n",
+      "| 1      | 0    | 0    | 0    | 0    |\n",
+      "| 2      | 5    | 6    | 7    | 8    |\n",
+      "| 3      | 10   | 12   | 14   | 16   |\n",
+      "| 4      | 15   | 18   | 21   | 24   |\n",
+      "| 5      | 20   | 24   | 28   | 32   |\n",
+      "| 6      | 25   | 30   | 35   | 40   |\n",
+      "| 7      | 30   | 36   | 42   | 48   |\n",
+      "| 8      | 35   | 42   | 49   | 56   |\n",
+      "| 9      | 40   | 48   | 56   | 64   |\n",
+      "| 10     | 45   | 54   | 63   | 72   |\n",
+      "| 11     | 50   | 60   | 70   | 80   |\n",
+      "| 12     | 55   | 66   | 77   | 88   |\n",
       "\n",
-      "| Region       | Sub-region      |\n",
-      "|--------------|-----------------|\n",
-      "| Lower 48     | Permian         |\n",
-      "|              | Eagle Ford      |\n",
-      "|              | Bakken          |\n",
-      "|              | Other           |\n",
-      "| Alaska       | GKA             |\n",
-      "|              | GWA             |\n",
-      "|              | GPA             |\n",
-      "|              | WNS             |\n",
-      "| EMENA        | Norway          |\n",
-      "|              | Libya           |\n",
-      "|              | Qatar           |\n",
-      "| Asia Pacific | APLNG           |\n",
-      "|              | Malaysia        |\n",
-      "|              | China           |\n",
-      "| Canada       | Montney         |\n",
-      "|              | Surmont         |\n",
-      "parsed_text: Our Differentiated Portfolio: Deep; Durable and Diverse\n",
-      "                              20 BBOE of Resource                                           Diverse Production Base\n",
-      "                            Under $40/BBL Cost of Supply                              10-Year Plan Cumulative Production (BBOE)\n",
-      "      S50                   S32/BBL                                                Lower 48                           Alaska\n",
-      "                    Average Cost of Supply\n",
-      "  3 $40                                                                                                                       GKA        GWA\n",
-      "                                                                                                                      GPA     WNS\n",
-      "      $30                                                                                                             EMENA\n",
-      "  3                                                                                                                              Norway\n",
-      "  8   $20\n",
-      "  E                                                                                                                   Qatar      Libya\n",
-      "                                                                                                                      Asia Pacific Canada\n",
-      "      $10                                                                          Permian\n",
-      "                                                                                                                      APLNG        Montney\n",
-      "       S0\n",
-      "                                               10              15              20                         Bakken\n",
-      "                                         Resource (BBOE)                           Eagle Ford             Other       Malaysia ChinaSurmont\n",
-      "                 Lower 48      Canada       Alaska      EMENA    Asia Pacific\n",
-      "Costs assumemid-cycle price environment of S60/BBL WTI:\n",
-      "                                                                                                                                           ConocoPhillips\n"
+      "~30% Improved Performance from 2019 to 2022\n",
+      "\n",
+      "## Delaware Basin Well Capex/EUR⁴ ($/BOE)\n",
+      "\n",
+      "| Company          | Capex/EUR |\n",
+      "|------------------|-----------|\n",
+      "| ConocoPhillips   | 10        |\n",
+      "| Competitor 1     | 15        |\n",
+      "| Competitor 2     | 20        |\n",
+      "| Competitor 3     | 25        |\n",
+      "| Competitor 4     | 30        |\n",
+      "| Competitor 5     | 35        |\n",
+      "| Competitor 6     | 40        |\n",
+      "| Competitor 7     | 45        |\n",
+      "\n",
+      "---\n",
+      "\n",
+      "¹ Unconventional acres.  \n",
+      "² Source: Enverus and ConocoPhillips (March 2023).  \n",
+      "³ Source: Enverus (March 2023) based on wells online year.  \n",
+      "⁴ Source: Enverus (March 2023). Average single well capex/EUR. Top eight public operators based on wells online in years 2021-2022, greater than 50% oil weight. COP based on COP well design. Competitors include: CVX, DVN, EOG, MTDR, OXY, PR and XOM.\n",
+      "parsed_text: \n",
+      "Delaware: Vast Inventory with Proven Track Record of Performance\n",
+      "        New                       Prolific Acreage Spanning Over                                                        12-Month Cumulative Production? (BOE/FT)\n",
+      "       Mexico                                 659,000 Net Acres'                                             40\n",
+      "                       Texas                                                                                                                                                                       3828\n",
+      "                                                                                                             30                                                                                    2019\n",
+      "                                                                                                             20                                                                            30%\n",
+      "                                                                                                             10                                                         Improved Performancefrom 2019 to 2022\n",
+      "           Total\n",
+      "   Permian Inventory\n",
+      "                    10-Year Operated\n",
+      "                                                                                                                            2                                                      10     11     12\n",
+      "                                                                                                                                                        Months\n",
+      "                                                                                                                         Delaware Basin Well Capex/EUR4 (S/BOE)\n",
+      "                      65%                                                                                     25\n",
+      "          Delaware Basin                                                                                      20\n",
+      "          Midland Basin                                                                                       15\n",
+      "                                              Low         HighCost of Supplyz                                 10  ConocoPhillips\n",
+      "     High Single-Digit Production Growth\n",
+      " \"Unconventional acres. 2Source: Enverus and ConocoPhillips (March 2023). 3SourceEnverus (March 2023) based on wells online year: \"Source; Enverus (March 2023). Average single well capex/EUR Top eight public operators based on\n",
+      "wells online in years 2021-2022, greater than 50% oil weight; COP based on COP well design: Competitors include; CVX DVN, EOG; MTDR, OXY, PR and XOM:                                     ConocoPhillips\n"
      ]
     }
    ],
    "source": [
-    "print(response.source_nodes[7].get_content(metadata_mode=\"all\"))"
+    "print(response.source_nodes[0].get_content(metadata_mode=\"all\"))"
    ]
   },
   {
@@ -921,20 +855,26 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "ConocoPhillips has a diverse production base spanning several key geographies:\n",
-      "\n",
-      "1. **Lower 48 (United States)**: This includes major production areas such as the Permian Basin, Bakken Formation, and Eagle Ford Shale.\n",
-      "2. **Alaska**: Significant operations in the North Slope region.\n",
-      "3. **Canada**: Operations in the Montney Formation and the Surmont oil sands project.\n",
-      "4. **EMENA (Europe, Middle East, and North Africa)**: Notable operations in Norway and Libya.\n",
-      "5. **Asia Pacific**: Includes operations in Malaysia, China, and the Australia Pacific LNG (APLNG) project.\n",
-      "6. **Qatar**: Involvement in the country's energy sector.\n",
-      "\n",
-      "These regions highlight the company's extensive and varied geographical footprint in the energy production industry.\n"
+      "Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
+      "=== Calling Function ===\n",
+      "Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
+      "=== Function Output ===\n",
+      "ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20/BOE.\n",
+      "=== Calling Function ===\n",
+      "Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
+      "=== Function Output ===\n",
+      "The average single well capex/EUR for competitors in the Delaware Basin is between $10 and $25 per BOE.\n",
+      "=== LLM Response ===\n",
+      "ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n",
+      "ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n"
      ]
     }
    ],
    "source": [
+    "# base_response = base_agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
+    "base_response = base_agent.query(\n",
+    "    \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
+    ")\n",
     "print(str(base_response))"
    ]
   },
@@ -948,31 +888,30 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Our Differentiated Portfolio: Deep; Durable and Diverse\n",
-      "                              20 BBOE of Resource                                           Diverse Production Base\n",
-      "                            Under $40/BBL Cost of Supply                              10-Year Plan Cumulative Production (BBOE)\n",
-      "      S50                   S32/BBL                                                Lower 48                           Alaska\n",
-      "                    Average Cost of Supply\n",
-      "  3 $40                                                                                                                       GKA        GWA\n",
-      "                                                                                                                      GPA     WNS\n",
-      "      $30                                                                                                             EMENA\n",
-      "  3                                                                                                                              Norway\n",
-      "  8   $20\n",
-      "  E                                                                                                                   Qatar      Libya\n",
-      "                                                                                                                      Asia Pacific Canada\n",
-      "      $10                                                                          Permian\n",
-      "                                                                                                                      APLNG        Montney\n",
-      "       S0\n",
-      "                                               10              15              20                         Bakken\n",
-      "                                         Resource (BBOE)                           Eagle Ford             Other       Malaysia ChinaSurmont\n",
-      "                 Lower 48      Canada       Alaska      EMENA    Asia Pacific\n",
-      "Costs assumemid-cycle price environment of S60/BBL WTI:\n",
-      "                                                                                                                                           ConocoPhillips\n"
+      "Deep, Durable and Diverse Portfolio with Significant Growth Runway\n",
+      "    1,2002022 Lower 48 Unconventional Production' (MBOED                                                  S50     ~S32/BBL\n",
+      "     000 ConocoPhillips                                                                                                   Cost of SupplyAverage\n",
+      "       00                                                                                                 S40\n",
+      "      500                                                                                            3\n",
+      "      400                                                                                            1    S30\n",
+      "     200\n",
+      "                                                                                                      5\n",
+      "   15,000ConocoPhillipsNet Remaining Well Inventory?                                                  1   S20\n",
+      "   12,000                                                                                                 S10\n",
+      "      000\n",
+      "      0o0                                                                                                  SO\n",
+      "     3,000                                                                                                                                                                         10\n",
+      "                                                                                                                                                Resource (BBOE)\n",
+      "                                                                                                                    Delaware Basin      Midland Basin      Eagle Ford       Bakken        Other\n",
+      "                        Largest Lower 48 Unconventional Producer; Growing into the Next Decade\n",
+      " onshore operated inventory that achieves 15% IRR at $SO/BBL WTI, Competitors include CVX, DVN, EOG, FANG, MRO, OXY, PXD,and XOM:\n",
+      "                                                                                                                                 Source: Wood Mackenzie Lower 48 Unconventional Plays 2022 ProductionCompetitors include CVX, DVN; EOG, FANG, MRO, OXY, PXD and XOM; greaterthan50% liquids weight: ?Source: Wood Mackenzie (March 2023), Lower 48\n",
+      "                                                                                                                                                                                  ConocoPhillips\n"
      ]
     }
    ],
    "source": [
-    "print(base_response.source_nodes[1].get_content(metadata_mode=\"all\"))"
+    "print(base_response.source_nodes[0].get_content(metadata_mode=\"llm\"))"
    ]
   }
  ],

examples/test_tesla_impact_report/test_gpt4o.ipynb

@@ -46,7 +46,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"<LLAMA_CLOUD_API_KEY>\""
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"<LLAMA_CLOUD_API_KEY>"
    ]
   },
   {

llama_parse/base.py

@@ -1,6 +1,6 @@
 import os
 import asyncio
-from urllib.parse import urlparse
+from io import TextIOWrapper
 
 import httpx
 import mimetypes
@@ -11,7 +11,8 @@ from contextlib import asynccontextmanager
 from io import BufferedIOBase
 
 from fsspec import AbstractFileSystem
-from llama_index.core.async_utils import asyncio_run, run_jobs
+from fsspec.spec import AbstractBufferedFile
+from llama_index.core.async_utils import run_jobs
 from llama_index.core.bridge.pydantic import Field, field_validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
@@ -90,14 +91,6 @@ class LlamaParse(BasePydanticReader):
         default=False,
         description="Note: Non compatible with gpt-4o. If set to true, the parser will use a faster mode to extract text from documents. This mode will skip OCR of images, and table/heading reconstruction.",
     )
-    premium_mode: bool = Field(
-        default=False,
-        description="Use our best parser mode if set to True.",
-    )
-    continuous_mode: bool = Field(
-        default=False,
-        description="Parse documents continuously, leading to better results on documents where tables span across two pages.",
-    )
     do_not_unroll_columns: Optional[bool] = Field(
         default=False,
         description="If set to true, the parser will keep column in the text according to document layout. Reduce reconstruction accuracy, and LLM's/embedings performances in most case.",
@@ -122,10 +115,6 @@ class LlamaParse(BasePydanticReader):
         default=None,
         description="The API key for the GPT-4o API. Lowers the cost of parsing.",
     )
-    guess_xlsx_sheet_names: Optional[bool] = Field(
-        default=False,
-        description="Whether to guess the sheet names of the xlsx file.",
-    )
     bounding_box: Optional[str] = Field(
         default=None,
         description="The bounding box to use to extract text from documents describe as a string containing the bounding box margins",
@@ -161,38 +150,6 @@ class LlamaParse(BasePydanticReader):
     custom_client: Optional[httpx.AsyncClient] = Field(
         default=None, description="A custom HTTPX client to use for sending requests."
     )
-    disable_ocr: bool = Field(
-        default=False,
-        description="Disable the OCR on the document. LlamaParse will only extract the copyable text from the document.",
-    )
-    is_formatting_instruction: bool = Field(
-        default=True,
-        description="Allow the parsing instruction to also format the output. Disable to have a cleaner markdown output.",
-    )
-    annotate_links: bool = Field(
-        default=False,
-        description="Annotate links found in the document to extract their URL.",
-    )
-    webhook_url: Optional[str] = Field(
-        default=None,
-        description="A URL that needs to be called at the end of the parsing job.",
-    )
-    azure_openai_deployment_name: Optional[str] = Field(
-        default=None, description="Azure Openai Deployment Name"
-    )
-    azure_openai_endpoint: Optional[str] = Field(
-        default=None, description="Azure Openai Endpoint"
-    )
-    azure_openai_api_version: Optional[str] = Field(
-        default=None, description="Azure Openai API Version"
-    )
-    azure_openai_key: Optional[str] = Field(
-        default=None, description="Azure Openai Key"
-    )
-    http_proxy: Optional[str] = Field(
-        default=None,
-        description="(optional) If set with input_url will use the specified http proxy to download the file.",
-    )
 
     @field_validator("api_key", mode="before", check_fields=True)
     @classmethod
@@ -224,28 +181,6 @@ class LlamaParse(BasePydanticReader):
             async with httpx.AsyncClient(timeout=self.max_timeout) as client:
                 yield client
 
-    def _is_input_url(self, file_path: FileInput) -> bool:
-        """Check if the input is a valid URL.
-
-        This method checks for:
-        - Proper URL scheme (http/https)
-        - Valid URL structure
-        - Network location (domain)
-        """
-        if not isinstance(file_path, str):
-            return False
-        try:
-            result = urlparse(file_path)
-            return all(
-                [
-                    result.scheme in ("http", "https"),
-                    result.netloc,  # Has domain
-                    result.scheme,  # Has scheme
-                ]
-            )
-        except Exception:
-            return False
-
     # upload a document and get back a job_id
     async def _create_job(
         self,
@@ -257,7 +192,6 @@ class LlamaParse(BasePydanticReader):
         url = f"{self.base_url}/api/parsing/upload"
         files = None
         file_handle = None
-        input_url = file_input if self._is_input_url(file_input) else None
 
         if isinstance(file_input, (bytes, BufferedIOBase)):
             if not extra_info or "file_name" not in extra_info:
@@ -267,8 +201,6 @@ class LlamaParse(BasePydanticReader):
             file_name = extra_info["file_name"]
             mime_type = mimetypes.guess_type(file_name)[0]
             files = {"file": (file_name, file_input, mime_type)}
-        elif input_url is not None:
-            files = None
         elif isinstance(file_input, (str, Path, PurePosixPath, PurePath)):
             file_path = str(file_input)
             file_ext = os.path.splitext(file_path)[1].lower()
@@ -295,8 +227,6 @@ class LlamaParse(BasePydanticReader):
             "skip_diagonal_text": self.skip_diagonal_text,
             "do_not_cache": self.do_not_cache,
             "fast_mode": self.fast_mode,
-            "premium_mode": self.premium_mode,
-            "continuous_mode": self.continuous_mode,
             "do_not_unroll_columns": self.do_not_unroll_columns,
             "gpt4o_mode": self.gpt4o_mode,
             "gpt4o_api_key": self.gpt4o_api_key,
@@ -304,11 +234,6 @@ class LlamaParse(BasePydanticReader):
             "use_vendor_multimodal_model": self.use_vendor_multimodal_model,
             "vendor_multimodal_model_name": self.vendor_multimodal_model_name,
             "take_screenshot": self.take_screenshot,
-            "disable_ocr": self.disable_ocr,
-            "guess_xlsx_sheet_names": self.guess_xlsx_sheet_names,
-            "is_formatting_instruction": self.is_formatting_instruction,
-            "annotate_links": self.annotate_links,
-            "from_python_package": True,
         }
 
         # only send page separator to server if it is not None
@@ -328,29 +253,6 @@ class LlamaParse(BasePydanticReader):
         if self.target_pages is not None:
             data["target_pages"] = self.target_pages
 
-        if self.webhook_url is not None:
-            data["webhook_url"] = self.webhook_url
-
-        # Azure OpenAI
-        if self.azure_openai_deployment_name is not None:
-            data["azure_openai_deployment_name"] = self.azure_openai_deployment_name
-
-        if self.azure_openai_endpoint is not None:
-            data["azure_openai_endpoint"] = self.azure_openai_endpoint
-
-        if self.azure_openai_api_version is not None:
-            data["azure_openai_api_version"] = self.azure_openai_api_version
-
-        if self.azure_openai_key is not None:
-            data["azure_openai_key"] = self.azure_openai_key
-
-        if input_url is not None:
-            files = None
-            data["input_url"] = str(input_url)
-
-        if self.http_proxy is not None:
-            data["http_proxy"] = self.http_proxy
-
         try:
             async with self.client_context() as client:
                 response = await client.post(
@@ -367,6 +269,12 @@ class LlamaParse(BasePydanticReader):
             if file_handle is not None:
                 file_handle.close()
 
+    @staticmethod
+    def __get_filename(f: Union[TextIOWrapper, AbstractBufferedFile]) -> str:
+        if isinstance(f, TextIOWrapper):
+            return f.name
+        return f.full_name
+
     async def _get_job_result(
         self, job_id: str, result_type: str, verbose: bool = False
     ) -> Dict[str, Any]:
@@ -395,8 +303,7 @@ class LlamaParse(BasePydanticReader):
                     continue
 
                 # Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
-                result_json = result.json()
-                status = result_json["status"]
+                status = result.json()["status"]
                 if status == "SUCCESS":
                     parsed_result = await client.get(result_url, headers=headers)
                     return parsed_result.json()
@@ -408,14 +315,6 @@ class LlamaParse(BasePydanticReader):
                         print(".", end="", flush=True)
 
                     await asyncio.sleep(self.check_interval)
-                else:
-                    error_code = result_json.get("error_code", "No error code found")
-                    error_message = result_json.get(
-                        "error_message", "No error message found"
-                    )
-
-                    exception_str = f"Job ID: {job_id} failed with status: {status}, Error code: {error_code}, Error message: {error_message}"
-                    raise Exception(exception_str)
 
     async def _aload_data(
         self,
@@ -460,7 +359,7 @@ class LlamaParse(BasePydanticReader):
         fs: Optional[AbstractFileSystem] = None,
     ) -> List[Document]:
         """Load data from the input path."""
-        if isinstance(file_path, (str, PurePosixPath, Path, bytes, BufferedIOBase)):
+        if isinstance(file_path, (str, Path, bytes, BufferedIOBase)):
             return await self._aload_data(
                 file_path, extra_info=extra_info, fs=fs, verbose=self.verbose
             )
@@ -502,7 +401,7 @@ class LlamaParse(BasePydanticReader):
     ) -> List[Document]:
         """Load data from the input path."""
         try:
-            return asyncio_run(self.aload_data(file_path, extra_info, fs=fs))
+            return asyncio.run(self.aload_data(file_path, extra_info, fs=fs))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
@@ -569,7 +468,7 @@ class LlamaParse(BasePydanticReader):
     ) -> List[dict]:
         """Parse the input path."""
         try:
-            return asyncio_run(self.aget_json(file_path, extra_info))
+            return asyncio.run(self.aget_json(file_path, extra_info))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
@@ -632,61 +531,7 @@ class LlamaParse(BasePydanticReader):
     def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
         """Download images from the parsed result."""
         try:
-            return asyncio_run(self.aget_images(json_result, download_path))
-        except RuntimeError as e:
-            if nest_asyncio_err in str(e):
-                raise RuntimeError(nest_asyncio_msg)
-            else:
-                raise e
-
-    async def aget_xlsx(
-        self, json_result: List[dict], download_path: str
-    ) -> List[dict]:
-        """Download images from the parsed result."""
-        headers = {"Authorization": f"Bearer {self.api_key}"}
-
-        # make the download path
-        if not os.path.exists(download_path):
-            os.makedirs(download_path)
-        try:
-            xlsx_list = []
-            for result in json_result:
-                job_id = result["job_id"]
-                if self.verbose:
-                    print("> XLSX")
-
-                xlsx_path = os.path.join(download_path, f"{job_id}.xlsx")
-
-                xlsx = {}
-
-                xlsx["path"] = xlsx_path
-                xlsx["job_id"] = job_id
-                xlsx["original_file_path"] = result.get("file_path", None)
-
-                with open(xlsx_path, "wb") as f:
-                    xlsx_url = (
-                        f"{self.base_url}/api/parsing/job/{job_id}/result/raw/xlsx"
-                    )
-                    async with self.client_context() as client:
-                        res = await client.get(
-                            xlsx_url, headers=headers, timeout=self.max_timeout
-                        )
-                        res.raise_for_status()
-                        f.write(res.content)
-                xlsx_list.append(xlsx)
-            return xlsx_list
-
-        except Exception as e:
-            print("Error while downloading xlsx:", e)
-            if self.ignore_errors:
-                return []
-            else:
-                raise e
-
-    def get_xlsx(self, json_result: List[dict], download_path: str) -> List[dict]:
-        """Download xlsx from the parsed result."""
-        try:
-            return asyncio_run(self.aget_xlsx(json_result, download_path))
+            return asyncio.run(self.aget_images(json_result, download_path))
         except RuntimeError as e:
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)

llama_parse/cli/main.py

@@ -1,92 +0,0 @@
-import click
-import json
-from enum import Enum
-from pathlib import Path
-from pydantic.fields import FieldInfo
-from typing import Any, Callable, List
-
-from llama_parse.base import LlamaParse
-
-
-def pydantic_field_to_click_option(name: str, field: FieldInfo) -> click.Option:
-    """Convert a Pydantic field to a Click option."""
-    kwargs = {
-        "default": field.default if field.default else None,
-        "help": field.description,
-    }
-
-    if isinstance(kwargs["default"], Enum):
-        kwargs["default"] = kwargs["default"].value
-
-    if field.annotation is bool:
-        kwargs["is_flag"] = True
-        if field.default and field.default is True:
-            name = f"no-{name}"
-    return click.option(f'--{name.replace("_", "-")}', **kwargs)
-
-
-def add_options(options: List[click.Option]) -> Callable:
-    def _add_options(func: Callable) -> Callable:
-        for option in reversed(options):
-            func = option(func)
-        return func
-
-    return _add_options
-
-
-@click.command()
-@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, path_type=Path))
-@click.option(
-    "--output-file", type=click.Path(path_type=Path), help="Path to save the output"
-)
-@click.option("--output-raw-json", is_flag=True, help="Output the raw JSON result")
-@add_options(
-    [
-        pydantic_field_to_click_option(name, field)
-        for name, field in LlamaParse.model_fields.items()
-        if name not in ["custom_client"]
-    ]
-)
-def parse(**kwargs: Any) -> None:
-    """Parse files using LlamaParse and output the results."""
-    file_paths = kwargs.pop("file_paths")
-    output_file = kwargs.pop("output_file")
-    output_raw_json = kwargs.pop("output_raw_json")
-
-    # Remove None values to use LlamaParse defaults
-    kwargs = {k: v for k, v in kwargs.items() if v is not None}
-
-    # Remove no- prefix for boolean flags
-    kwargs = {k.replace("no_", ""): v for k, v in kwargs.items()}
-
-    parser = LlamaParse(**kwargs)
-    if output_raw_json:
-        results = parser.get_json_result(list(file_paths))
-
-        if output_file:
-            with output_file.open("w") as f:
-                json.dump(results, f)
-            click.echo(f"Results saved to {output_file}")
-        else:
-            click.echo(results)
-    else:
-        results = parser.load_data(list(file_paths))
-
-        if output_file:
-            with output_file.open("w") as f:
-                for i, doc in enumerate(results):
-                    f.write(f"File: {doc.metadata.get('file_path', 'Unknown')}\n")  # type: ignore
-                    f.write(doc.text)  # type: ignore
-                    if i < len(results) - 1:
-                        f.write("\n\n---\n\n")
-            click.echo(f"Results saved to {output_file}")
-        else:
-            for i, doc in enumerate(results):
-                click.echo(f"File: {doc.metadata.get('file_path', 'Unknown')}")  # type: ignore
-                click.echo(doc.text)  # type: ignore
-                if i < len(results) - 1:
-                    click.echo("\n---\n")
-
-
-if __name__ == "__main__":
-    parse()

llama_parse/utils.py

@@ -10,6 +10,7 @@ class ResultType(str, Enum):
 
     TXT = "text"
     MD = "markdown"
+    JSON = "json"
 
 
 class Language(str, Enum):

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "llama-parse"
-version = "0.5.14"
+version = "0.5.5"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"
@@ -14,11 +14,7 @@ packages = [{include = "llama_parse"}]
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
 llama-index-core = ">=0.11.0"
-click = "^8.1.7"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
 ipykernel = "^6.29.0"
-
-[tool.poetry.scripts]
-llama-parse = "llama_parse.cli.main:parse"

tests/test_reader.py

@@ -76,14 +76,13 @@ def test_simple_page_markdown_buffer(markdown_parser: LlamaParse) -> None:
     os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
     reason="LLAMA_CLOUD_API_KEY not set",
 )
-@pytest.mark.asyncio
-async def test_simple_page_with_custom_fs() -> None:
+def test_simple_page_with_custom_fs() -> None:
     parser = LlamaParse(result_type="markdown")
     fs = LocalFileSystem()
     filepath = os.path.join(
         os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
     )
-    result = await parser.aload_data(filepath, fs=fs)
+    result = parser.load_data(filepath, fs=fs)
     assert len(result) == 1
 
 
@@ -91,14 +90,13 @@ async def test_simple_page_with_custom_fs() -> None:
     os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
     reason="LLAMA_CLOUD_API_KEY not set",
 )
-@pytest.mark.asyncio
-async def test_simple_page_progress_workers() -> None:
+def test_simple_page_progress_workers() -> None:
     parser = LlamaParse(result_type="markdown", show_progress=True, verbose=True)
 
     filepath = os.path.join(
         os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
     )
-    result = await parser.aload_data([filepath, filepath])
+    result = parser.load_data([filepath, filepath])
     assert len(result) == 2
     assert len(result[0].text) > 0
 
@@ -109,7 +107,7 @@ async def test_simple_page_progress_workers() -> None:
     filepath = os.path.join(
         os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
     )
-    result = await parser.aload_data([filepath, filepath])
+    result = parser.load_data([filepath, filepath])
     assert len(result) == 2
     assert len(result[0].text) > 0
 
@@ -118,59 +116,12 @@ async def test_simple_page_progress_workers() -> None:
     os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
     reason="LLAMA_CLOUD_API_KEY not set",
 )
-@pytest.mark.asyncio
-async def test_custom_client() -> None:
+def test_custom_client() -> None:
     custom_client = AsyncClient(verify=False, timeout=10)
     parser = LlamaParse(result_type="markdown", custom_client=custom_client)
     filepath = os.path.join(
         os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
     )
-    result = await parser.aload_data(filepath)
+    result = parser.load_data(filepath)
     assert len(result) == 1
     assert len(result[0].text) > 0
-
-
-@pytest.mark.skipif(
-    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
-    reason="LLAMA_CLOUD_API_KEY not set",
-)
-@pytest.mark.asyncio
-async def test_input_url() -> None:
-    parser = LlamaParse(result_type="markdown")
-
-    # links to a resume example
-    input_url = "https://cdn-blog.novoresume.com/articles/google-docs-resume-templates/basic-google-docs-resume.png"
-    result = await parser.aload_data(input_url)
-    assert len(result) == 1
-    assert "your name" in result[0].text.lower()
-
-
-@pytest.mark.skipif(
-    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
-    reason="LLAMA_CLOUD_API_KEY not set",
-)
-@pytest.mark.asyncio
-async def test_input_url_with_website_input() -> None:
-    parser = LlamaParse(result_type="markdown")
-    input_url = "https://www.google.com"
-    result = await parser.aload_data(input_url)
-    assert len(result) == 1
-    assert "google" in result[0].text.lower()
-
-
-@pytest.mark.skipif(
-    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
-    reason="LLAMA_CLOUD_API_KEY not set",
-)
-@pytest.mark.asyncio
-async def test_mixing_input_types() -> None:
-    parser = LlamaParse(result_type="markdown")
-    filepath = os.path.join(
-        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
-    )
-    input_url = "https://www.google.com"
-    result = await parser.aload_data([filepath, input_url])
-
-    assert len(result) == 2
-    assert "table 2" in result[0].text.lower()
-    assert "google" in result[1].text.lower()