mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-25 16:15:24 -04:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jerry Liu
|
d9d39b8138 | ||
|
Jerry Liu
|
7dbbc8b44d |
@@ -2,4 +2,3 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.DS_Store
|
||||
.idea
|
||||
|
||||
@@ -1,29 +1,16 @@
|
||||
# LlamaParse
|
||||
|
||||
[](https://pypi.org/project/llama-parse/)
|
||||
[](https://github.com/run-llama/llama_parse/graphs/contributors)
|
||||
[](https://discord.gg/dGcwcsnxhU)
|
||||
|
||||
LlamaParse is a **GenAI-native document parser** that can parse complex document data for any downstream LLM use case (RAG, agents).
|
||||
|
||||
It is really good at the following:
|
||||
|
||||
- ✅ **Broad file type support**: Parsing a variety of unstructured file types (.pdf, .pptx, .docx, .xlsx, .html) with text, tables, visual elements, weird layouts, and more.
|
||||
- ✅ **Table recognition**: Parsing embedded tables accurately into text and semi-structured representations.
|
||||
- ✅ **Multimodal parsing and chunking**: Extracting visual elements (images/diagrams) into structured formats and return image chunks using the latest multimodal models.
|
||||
- ✅ **Custom parsing**: Input custom prompt instructions to customize the output the way you want it.
|
||||
LlamaParse is an API created by LlamaIndex to efficiently parse and represent files for efficient retrieval and context augmentation using LlamaIndex frameworks.
|
||||
|
||||
LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).
|
||||
|
||||
The free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page by default. There is a sandbox available to test the API [**https://cloud.llamaindex.ai/parse ↗**](https://cloud.llamaindex.ai/parse).
|
||||
Free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page.
|
||||
|
||||
Read below for some quickstart information, or see the [full documentation](https://docs.cloud.llamaindex.ai/).
|
||||
|
||||
If you're a company interested in enterprise RAG solutions, and/or high volume/on-prem usage of LlamaParse, come [talk to us](https://www.llamaindex.ai/contact).
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, login and get an api-key from [**https://cloud.llamaindex.ai/api-key ↗**](https://cloud.llamaindex.ai/api-key).
|
||||
First, login and get an api-key from [**https://cloud.llamaindex.ai ↗**](https://cloud.llamaindex.ai).
|
||||
|
||||
Then, make sure you have the latest LlamaIndex version installed.
|
||||
|
||||
@@ -87,18 +74,13 @@ parser = LlamaParse(
|
||||
language="en", # Optionally you can define a language, default=en
|
||||
)
|
||||
|
||||
file_name = "my_file1.pdf"
|
||||
extra_info = {"file_name": file_name}
|
||||
|
||||
with open(f"./{file_name}", "rb") as f:
|
||||
# must provide extra_info with file_name key with passing file object
|
||||
documents = parser.load_data(f, extra_info=extra_info)
|
||||
with open("./my_file1.pdf", "rb") as f:
|
||||
documents = parser.load_data(f)
|
||||
|
||||
# you can also pass file bytes directly
|
||||
with open(f"./{file_name}", "rb") as f:
|
||||
with open("./my_file1.pdf", "rb") as f:
|
||||
file_bytes = f.read()
|
||||
# must provide extra_info with file_name key with passing file bytes
|
||||
documents = parser.load_data(file_bytes, extra_info=extra_info)
|
||||
documents = parser.load_data(file_bytes)
|
||||
```
|
||||
|
||||
## Using with `SimpleDirectoryReader`
|
||||
@@ -142,9 +124,3 @@ Several end-to-end indexing examples can be found in the examples folder
|
||||
## Terms of Service
|
||||
|
||||
See the [Terms of Service Here](./TOS.pdf).
|
||||
|
||||
## Get in Touch (LlamaCloud)
|
||||
|
||||
LlamaParse is part of LlamaCloud, our e2e enterprise RAG platform that provides out-of-the-box, production-ready connectors, indexing, and retrieval over your complex data sources. We offer SaaS and VPC options.
|
||||
|
||||
LlamaCloud is currently available via waitlist (join by [creating an account](https://cloud.llamaindex.ai/)). If you're interested in state-of-the-art quality and in centralizing your RAG efforts, come [get in touch with us](https://www.llamaindex.ai/contact).
|
||||
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
|
Before Width: | Height: | Size: 195 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 363 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 343 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 185 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 254 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 650 KiB |
@@ -4,9 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Building a Multimodal RAG Pipeline over an Auto Insurance Claim\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/multimodal/insurance_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
"# Building a Multimodal RAG Pipeline over an Auto Insurance Claim"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
"source": [
|
||||
"# Building a RAG Pipeline over Legal Documents\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/multimodal/legal_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This example shows how LlamaParse and LlamaIndex can be used to parse various types of legal documents, which may contain complex tabular data. The advantage of this is being able to quickly retrieve a specific answer to a legal question with comprehensive context — knowledge of precedents, statutes, and cases presented in the given documents. A user can quickly find the answer to or find out more details about a specific legal question without having to read through the often long documents by using LLMs.\n",
|
||||
"\n",
|
||||
"In this example, we will be using legal documents from the archive of the Library of Congress ([link to dataset](https://www.loc.gov/item/2020445568/)). These documents vary by format, with some containing pure text and others containing headings, sections, and large tables. This shows how LlamaParse can parse a wide variety of documents and still retrieve accurate results.\n",
|
||||
@@ -165,18 +165,7 @@
|
||||
"execution_count": null,
|
||||
"id": "ef82a985-4088-4bb7-9a21-0318e1b9207d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Parsing text...\n",
|
||||
"Started parsing the file under job_id 62f157a9-9ef9-4e5b-95ac-67093fa25800\n",
|
||||
"..........Parsing PDF file...\n",
|
||||
"Started parsing the file under job_id 1ddd5654-062b-4e19-b488-d66efc9c509d\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(f\"Parsing text...\")\n",
|
||||
"docs_text = parser_text.load_data(\"data/conocophillips.pdf\")\n",
|
||||
@@ -185,36 +174,42 @@
|
||||
"md_json_list = md_json_objs[0][\"pages\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7506b603-c01f-45de-b354-4a0728dde03c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(docs_text[0].get_content())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5318fb7b-fe6a-4a8a-b82e-4ed7b4512c37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(md_json_list[10][\"md\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7a46a73e-c6e2-4b0b-bd10-31b0d3e4b70f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"# Commitment to Disciplined Reinvestment Rate\n",
|
||||
"\n",
|
||||
"| Period | Description | Reinvestment Rate | WTI Average |\n",
|
||||
"|--------------|--------------------------------------|-------------------|-------------|\n",
|
||||
"| 2012-2016 | Industry Growth Focus | >100% | ~$75/BBL |\n",
|
||||
"| 2017-2022 | ConocoPhillips Strategy Reset | <60% | ~$63/BBL |\n",
|
||||
"| 2023E | | | at $80/BBL |\n",
|
||||
"| 2024-2028 | Disciplined Reinvestment Rate | ~50% | at $60/BBL |\n",
|
||||
"| 2029-2032 | | ~6% CFO CAGR | at $60/BBL |\n",
|
||||
"\n",
|
||||
"- **Historic Reinvestment Rate**: Gray bars\n",
|
||||
"- **Reinvestment Rate at $60/BBL WTI**: Blue bars\n",
|
||||
"- **Reinvestment Rate at $80/BBL WTI**: Dashed blue lines\n",
|
||||
"\n",
|
||||
"Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n"
|
||||
"dict_keys(['page', 'text', 'md', 'images', 'items'])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(md_json_list[10][\"md\"])"
|
||||
"print(md_json_list[1].keys())"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -304,7 +299,7 @@
|
||||
" image_files = _get_sorted_image_files(image_dir) if image_dir is not None else None\n",
|
||||
" md_texts = [d[\"md\"] for d in json_dicts] if json_dicts is not None else None\n",
|
||||
"\n",
|
||||
" doc_chunks = [c for d in docs for c in d.text.split(\"---\")]\n",
|
||||
" doc_chunks = docs[0].text.split(\"---\")\n",
|
||||
" for idx, doc_chunk in enumerate(doc_chunks):\n",
|
||||
" chunk_metadata = {\"page_num\": idx + 1}\n",
|
||||
" if image_files is not None:\n",
|
||||
@@ -344,23 +339,25 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page_num: 11\n",
|
||||
"image_path: data_images/1ddd5654-062b-4e19-b488-d66efc9c509d-page_39.jpg\n",
|
||||
"image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-10.jpg\n",
|
||||
"parsed_text_markdown: # Commitment to Disciplined Reinvestment Rate\n",
|
||||
"\n",
|
||||
"| Period | Description | Reinvestment Rate | WTI Average |\n",
|
||||
"|--------------|--------------------------------------|-------------------|-------------|\n",
|
||||
"| 2012-2016 | Industry Growth Focus | >100% | ~$75/BBL |\n",
|
||||
"| 2017-2022 | ConocoPhillips Strategy Reset | <60% | ~$63/BBL |\n",
|
||||
"| 2023E | | | at $80/BBL |\n",
|
||||
"| 2024-2028 | Disciplined Reinvestment Rate | ~50% | at $60/BBL |\n",
|
||||
"| 2029-2032 | | ~6% CFO CAGR | at $60/BBL |\n",
|
||||
"| Year | Reinvestment Rate | WTI Average Price | Reinvestment Rate at $60/BBL WTI | Reinvestment Rate at $80/BBL WTI |\n",
|
||||
"|------------|-------------------|-------------------|----------------------------------|----------------------------------|\n",
|
||||
"| 2012-2016 | >100% | ~$75/BBL | | |\n",
|
||||
"| 2017-2022 | <60% | ~$63/BBL | | |\n",
|
||||
"| 2023E | | | | at $80/BBL WTI |\n",
|
||||
"| 2024-2028 | | | at $60/BBL WTI | at $80/BBL WTI |\n",
|
||||
"| 2029-2032 | | | at $60/BBL WTI | at $80/BBL WTI |\n",
|
||||
"\n",
|
||||
"- **Historic Reinvestment Rate**: Gray bars\n",
|
||||
"- **Reinvestment Rate at $60/BBL WTI**: Blue bars\n",
|
||||
"- **Reinvestment Rate at $80/BBL WTI**: Dashed blue lines\n",
|
||||
"**Disciplined Reinvestment Rate is the Foundation for Superior Returns on and of Capital, while Driving Durable CFO Growth**\n",
|
||||
"\n",
|
||||
"Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n",
|
||||
"parsed_text: Commitment to Disciplined Reinvestment Rate\n",
|
||||
"- ~50% 10-Year Reinvestment Rate\n",
|
||||
"- ~6% CFO CAGR 2024-2032 at $60/BBL WTI Mid-Cycle Planning Price\n",
|
||||
"\n",
|
||||
"**Note:** Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n",
|
||||
"parsed_text: \n",
|
||||
"Commitment to Disciplined Reinvestment Rate\n",
|
||||
" Industry ConocoPhillips\n",
|
||||
" Strategy Reset Disciplined Reinvestment Rate is the Foundation for Superior\n",
|
||||
" Growth Focus Returns on and of Capital, while Driving Durable CFO Growth\n",
|
||||
@@ -377,7 +374,7 @@
|
||||
" 0%\n",
|
||||
" 2012-2016 2017-2022 2023E 2024-2028 2029-2032\n",
|
||||
" Historic Reinvestment Rate Reinvestment Rate at $60/BBL WTI Reinvestment Rate at $80/BBL WTI\n",
|
||||
" Reinvestment rate and cash from operations (CFO) are non-GAAP measures: Definitions and reconciliations are included in the Appendix ConocoPhillips\n"
|
||||
" Reinvestment rate andcashfrom operations (CFO) are non-GAAP measures: Definitions and reconciliations are included in the Appendix ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -400,17 +397,7 @@
|
||||
"execution_count": null,
|
||||
"id": "6ea53c31-0e38-421c-8d9b-0e3adaa1677e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/Users/jerryliu/Programming/gpt_index/.venv/lib/python3.10/site-packages/tiktoken/core.py:50: RuntimeWarning: coroutine 'LlamaParse.aload_data' was never awaited\n",
|
||||
" self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str)\n",
|
||||
"RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from llama_index.core import (\n",
|
||||
@@ -601,7 +588,7 @@
|
||||
" Under $40/BBL Cost of Supply 10-Year Plan Cumulative Production (BBOE)\n",
|
||||
" S50 S32/BBL Lower 48 Alaska\n",
|
||||
" Average Cost of Supply\n",
|
||||
" 3 $40 GKA GWA\n",
|
||||
" 3$40 GKA GWA\n",
|
||||
" GPA WNS\n",
|
||||
" $30 EMENA\n",
|
||||
" 3 Norway\n",
|
||||
@@ -612,7 +599,7 @@
|
||||
" APLNG Montney\n",
|
||||
" S0\n",
|
||||
" 10 15 20 Bakken\n",
|
||||
" Resource (BBOE) Eagle Ford Other Malaysia ChinaSurmont\n",
|
||||
" Resource (BBOE) Eagle Ford Other MalaysiaChina Surmont\n",
|
||||
" Lower 48 Canada Alaska EMENA Asia Pacific\n",
|
||||
"Costs assumemid-cycle price environment of S60/BBL WTI:\n",
|
||||
" ConocoPhillips\n"
|
||||
@@ -700,126 +687,70 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d78e53cf-35cb-4ef8-b03e-1b47ba15ae64",
|
||||
"id": "1cdce5d8-6bb3-4cd3-929d-1cec249d9052",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Added user message to memory: Tell me about the diverse geographies where Conoco Phillips has a production base\n",
|
||||
"Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"Conoco Phillips production base geographies\"}\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"ConocoPhillips' production base geographies include:\n",
|
||||
"The ConocoPhillips capex/EUR in the Delaware Basin is $10/BOE.\n",
|
||||
"\n",
|
||||
"1. **Lower 48** (Permian, Eagle Ford, Bakken, Other)\n",
|
||||
"2. **Alaska** (GKA, GWA, GPA, WNS)\n",
|
||||
"3. **EMENA** (Norway, Libya, Qatar)\n",
|
||||
"4. **Asia Pacific** (APLNG, Malaysia, China)\n",
|
||||
"5. **Canada** (Montney, Surmont)\n",
|
||||
"\n",
|
||||
"This information was derived from the image on page 14, which provides a detailed breakdown of the diverse production base and the regions involved. The parsed markdown and raw text also support this information, but the image provides the clearest and most comprehensive view. There are no discrepancies between the image and the parsed text in this case.\n",
|
||||
"=== LLM Response ===\n",
|
||||
"ConocoPhillips has a diverse production base spread across various geographies, including:\n",
|
||||
"\n",
|
||||
"1. **Lower 48**:\n",
|
||||
" - Permian Basin\n",
|
||||
" - Eagle Ford\n",
|
||||
" - Bakken\n",
|
||||
" - Other regions within the continental United States\n",
|
||||
"\n",
|
||||
"2. **Alaska**:\n",
|
||||
" - Greater Kuparuk Area (GKA)\n",
|
||||
" - Greater Prudhoe Area (GPA)\n",
|
||||
" - Greater Willow Area (GWA)\n",
|
||||
" - Western North Slope (WNS)\n",
|
||||
"\n",
|
||||
"3. **EMENA (Europe, Middle East, and North Africa)**:\n",
|
||||
" - Norway\n",
|
||||
" - Libya\n",
|
||||
" - Qatar\n",
|
||||
"\n",
|
||||
"4. **Asia Pacific**:\n",
|
||||
" - Australia Pacific LNG (APLNG)\n",
|
||||
" - Malaysia\n",
|
||||
" - China\n",
|
||||
"\n",
|
||||
"5. **Canada**:\n",
|
||||
" - Montney\n",
|
||||
" - Surmont\n",
|
||||
"\n",
|
||||
"These regions highlight the global reach and diverse geographical footprint of ConocoPhillips' production operations.\n",
|
||||
"Added user message to memory: Tell me about the diverse geographies where Conoco Phillips has a production base\n",
|
||||
"I obtained this information from the image provided. The image clearly shows a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\" where ConocoPhillips is listed with a capex/EUR of $10/BOE. This information is consistent with the parsed markdown text, which also lists ConocoPhillips' capex/EUR as $10/BOE in the Delaware Basin. There are no discrepancies between the image and the parsed markdown text in this case.\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"diverse geographies where Conoco Phillips has a production base\"}\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"ConocoPhillips has a diverse production base that includes the Lower 48 (Permian, Bakken, Eagle Ford), Alaska, Canada (Montney, Surmont), EMENA (Norway, Libya), Asia Pacific (Malaysia, China, APLNG), and Qatar.\n",
|
||||
"The competitors' Capex/EUR in the Delaware Basin can be found in the image on the slide titled \"Delaware: Vast Inventory with Proven Track Record of Performance.\" The relevant information is presented in a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\".\n",
|
||||
"\n",
|
||||
"Here are the details:\n",
|
||||
"\n",
|
||||
"- ConocoPhillips: $10/BOE\n",
|
||||
"- Competitor 1: $15/BOE\n",
|
||||
"- Competitor 2: $20/BOE\n",
|
||||
"- Competitor 3: $25/BOE\n",
|
||||
"- Competitor 4: $30/BOE\n",
|
||||
"- Competitor 5: $35/BOE\n",
|
||||
"- Competitor 6: $40/BOE\n",
|
||||
"- Competitor 7: $45/BOE\n",
|
||||
"\n",
|
||||
"This information was obtained directly from the image, which provides a clear visual representation of the Capex/EUR values for ConocoPhillips and its competitors in the Delaware Basin. The parsed markdown text also confirms these values, ensuring consistency between the image and the text.\n",
|
||||
"=== LLM Response ===\n",
|
||||
"ConocoPhillips has a diverse production base spanning several key geographies:\n",
|
||||
"The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
|
||||
"\n",
|
||||
"1. **Lower 48 (United States)**: This includes major production areas such as the Permian Basin, Bakken Formation, and Eagle Ford Shale.\n",
|
||||
"2. **Alaska**: Significant operations in the North Slope region.\n",
|
||||
"3. **Canada**: Operations in the Montney Formation and the Surmont oil sands project.\n",
|
||||
"4. **EMENA (Europe, Middle East, and North Africa)**: Notable operations in Norway and Libya.\n",
|
||||
"5. **Asia Pacific**: Includes operations in Malaysia, China, and the Australia Pacific LNG (APLNG) project.\n",
|
||||
"6. **Qatar**: Involvement in the country's energy sector.\n",
|
||||
"- **ConocoPhillips**: $10/BOE\n",
|
||||
"- **Competitor 1**: $15/BOE\n",
|
||||
"- **Competitor 2**: $20/BOE\n",
|
||||
"- **Competitor 3**: $25/BOE\n",
|
||||
"- **Competitor 4**: $30/BOE\n",
|
||||
"- **Competitor 5**: $35/BOE\n",
|
||||
"- **Competitor 6**: $40/BOE\n",
|
||||
"- **Competitor 7**: $45/BOE\n",
|
||||
"\n",
|
||||
"These regions highlight the company's extensive and varied geographical footprint in the energy production industry.\n"
|
||||
"This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n",
|
||||
"The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
|
||||
"\n",
|
||||
"- **ConocoPhillips**: $10/BOE\n",
|
||||
"- **Competitor 1**: $15/BOE\n",
|
||||
"- **Competitor 2**: $20/BOE\n",
|
||||
"- **Competitor 3**: $25/BOE\n",
|
||||
"- **Competitor 4**: $30/BOE\n",
|
||||
"- **Competitor 5**: $35/BOE\n",
|
||||
"- **Competitor 6**: $40/BOE\n",
|
||||
"- **Competitor 7**: $45/BOE\n",
|
||||
"\n",
|
||||
"This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query = (\n",
|
||||
" \"Tell me about the diverse geographies where Conoco Phillips has a production base\"\n",
|
||||
"# response = agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
|
||||
"response = agent.query(\n",
|
||||
" \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
|
||||
")\n",
|
||||
"response = agent.query(query)\n",
|
||||
"base_response = base_agent.query(query)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "355d2aa4-c26f-480e-b512-4446acbd9227",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"ConocoPhillips has a diverse production base spread across various geographies, including:\n",
|
||||
"\n",
|
||||
"1. **Lower 48**:\n",
|
||||
" - Permian Basin\n",
|
||||
" - Eagle Ford\n",
|
||||
" - Bakken\n",
|
||||
" - Other regions within the continental United States\n",
|
||||
"\n",
|
||||
"2. **Alaska**:\n",
|
||||
" - Greater Kuparuk Area (GKA)\n",
|
||||
" - Greater Prudhoe Area (GPA)\n",
|
||||
" - Greater Willow Area (GWA)\n",
|
||||
" - Western North Slope (WNS)\n",
|
||||
"\n",
|
||||
"3. **EMENA (Europe, Middle East, and North Africa)**:\n",
|
||||
" - Norway\n",
|
||||
" - Libya\n",
|
||||
" - Qatar\n",
|
||||
"\n",
|
||||
"4. **Asia Pacific**:\n",
|
||||
" - Australia Pacific LNG (APLNG)\n",
|
||||
" - Malaysia\n",
|
||||
" - China\n",
|
||||
"\n",
|
||||
"5. **Canada**:\n",
|
||||
" - Montney\n",
|
||||
" - Surmont\n",
|
||||
"\n",
|
||||
"These regions highlight the global reach and diverse geographical footprint of ConocoPhillips' production operations.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(str(response))"
|
||||
]
|
||||
},
|
||||
@@ -833,82 +764,85 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page_num: 14\n",
|
||||
"image_path: data_images/1ddd5654-062b-4e19-b488-d66efc9c509d-page_12.jpg\n",
|
||||
"parsed_text_markdown: # Our Differentiated Portfolio: Deep, Durable and Diverse\n",
|
||||
"page_num: 38\n",
|
||||
"image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-37.jpg\n",
|
||||
"parsed_text_markdown: # Delaware: Vast Inventory with Proven Track Record of Performance\n",
|
||||
"\n",
|
||||
"## ~20 BBOE of Resource\n",
|
||||
"Under $40/BBL Cost of Supply\n",
|
||||
"## Prolific Acreage Spanning Over ~659,000 Net Acres¹\n",
|
||||
"\n",
|
||||
"### ~ $32/BBL\n",
|
||||
"Average Cost of Supply\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### WTI Cost of Supply ($/BBL)\n",
|
||||
"### Total 10-Year Operated Permian Inventory\n",
|
||||
"\n",
|
||||
"| Cost ($/BBL) | Resource (BBOE) |\n",
|
||||
"|--------------|-----------------|\n",
|
||||
"| $0 | 0 |\n",
|
||||
"| $10 | |\n",
|
||||
"| $20 | |\n",
|
||||
"| $30 | |\n",
|
||||
"| $40 | |\n",
|
||||
"| $50 | |\n",
|
||||
"- Delaware Basin: 65%\n",
|
||||
"- Midland Basin: 35%\n",
|
||||
"\n",
|
||||
"- **Legend:**\n",
|
||||
" - Lower 48\n",
|
||||
" - Canada\n",
|
||||
" - Alaska\n",
|
||||
" - EMENA\n",
|
||||
" - Asia Pacific\n",
|
||||
"### High Single-Digit Production Growth\n",
|
||||
"\n",
|
||||
"*Costs assume a mid-cycle price environment of $60/BBL WTI.*\n",
|
||||
"## 12-Month Cumulative Production³ (BOE/FT)\n",
|
||||
"\n",
|
||||
"## Diverse Production Base\n",
|
||||
"10-Year Plan Cumulative Production (BBOE)\n",
|
||||
"| Months | 2019 | 2020 | 2021 | 2022 |\n",
|
||||
"|--------|------|------|------|------|\n",
|
||||
"| 1 | 0 | 0 | 0 | 0 |\n",
|
||||
"| 2 | 5 | 6 | 7 | 8 |\n",
|
||||
"| 3 | 10 | 12 | 14 | 16 |\n",
|
||||
"| 4 | 15 | 18 | 21 | 24 |\n",
|
||||
"| 5 | 20 | 24 | 28 | 32 |\n",
|
||||
"| 6 | 25 | 30 | 35 | 40 |\n",
|
||||
"| 7 | 30 | 36 | 42 | 48 |\n",
|
||||
"| 8 | 35 | 42 | 49 | 56 |\n",
|
||||
"| 9 | 40 | 48 | 56 | 64 |\n",
|
||||
"| 10 | 45 | 54 | 63 | 72 |\n",
|
||||
"| 11 | 50 | 60 | 70 | 80 |\n",
|
||||
"| 12 | 55 | 66 | 77 | 88 |\n",
|
||||
"\n",
|
||||
"| Region | Sub-region |\n",
|
||||
"|--------------|-----------------|\n",
|
||||
"| Lower 48 | Permian |\n",
|
||||
"| | Eagle Ford |\n",
|
||||
"| | Bakken |\n",
|
||||
"| | Other |\n",
|
||||
"| Alaska | GKA |\n",
|
||||
"| | GWA |\n",
|
||||
"| | GPA |\n",
|
||||
"| | WNS |\n",
|
||||
"| EMENA | Norway |\n",
|
||||
"| | Libya |\n",
|
||||
"| | Qatar |\n",
|
||||
"| Asia Pacific | APLNG |\n",
|
||||
"| | Malaysia |\n",
|
||||
"| | China |\n",
|
||||
"| Canada | Montney |\n",
|
||||
"| | Surmont |\n",
|
||||
"parsed_text: Our Differentiated Portfolio: Deep; Durable and Diverse\n",
|
||||
" 20 BBOE of Resource Diverse Production Base\n",
|
||||
" Under $40/BBL Cost of Supply 10-Year Plan Cumulative Production (BBOE)\n",
|
||||
" S50 S32/BBL Lower 48 Alaska\n",
|
||||
" Average Cost of Supply\n",
|
||||
" 3 $40 GKA GWA\n",
|
||||
" GPA WNS\n",
|
||||
" $30 EMENA\n",
|
||||
" 3 Norway\n",
|
||||
" 8 $20\n",
|
||||
" E Qatar Libya\n",
|
||||
" Asia Pacific Canada\n",
|
||||
" $10 Permian\n",
|
||||
" APLNG Montney\n",
|
||||
" S0\n",
|
||||
" 10 15 20 Bakken\n",
|
||||
" Resource (BBOE) Eagle Ford Other Malaysia ChinaSurmont\n",
|
||||
" Lower 48 Canada Alaska EMENA Asia Pacific\n",
|
||||
"Costs assumemid-cycle price environment of S60/BBL WTI:\n",
|
||||
" ConocoPhillips\n"
|
||||
"~30% Improved Performance from 2019 to 2022\n",
|
||||
"\n",
|
||||
"## Delaware Basin Well Capex/EUR⁴ ($/BOE)\n",
|
||||
"\n",
|
||||
"| Company | Capex/EUR |\n",
|
||||
"|------------------|-----------|\n",
|
||||
"| ConocoPhillips | 10 |\n",
|
||||
"| Competitor 1 | 15 |\n",
|
||||
"| Competitor 2 | 20 |\n",
|
||||
"| Competitor 3 | 25 |\n",
|
||||
"| Competitor 4 | 30 |\n",
|
||||
"| Competitor 5 | 35 |\n",
|
||||
"| Competitor 6 | 40 |\n",
|
||||
"| Competitor 7 | 45 |\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"¹ Unconventional acres. \n",
|
||||
"² Source: Enverus and ConocoPhillips (March 2023). \n",
|
||||
"³ Source: Enverus (March 2023) based on wells online year. \n",
|
||||
"⁴ Source: Enverus (March 2023). Average single well capex/EUR. Top eight public operators based on wells online in years 2021-2022, greater than 50% oil weight. COP based on COP well design. Competitors include: CVX, DVN, EOG, MTDR, OXY, PR and XOM.\n",
|
||||
"parsed_text: \n",
|
||||
"Delaware: Vast Inventory with Proven Track Record of Performance\n",
|
||||
" New Prolific Acreage Spanning Over 12-Month Cumulative Production? (BOE/FT)\n",
|
||||
" Mexico 659,000 Net Acres' 40\n",
|
||||
" Texas 3828\n",
|
||||
" 30 2019\n",
|
||||
" 20 30%\n",
|
||||
" 10 Improved Performancefrom 2019 to 2022\n",
|
||||
" Total\n",
|
||||
" Permian Inventory\n",
|
||||
" 10-Year Operated\n",
|
||||
" 2 10 11 12\n",
|
||||
" Months\n",
|
||||
" Delaware Basin Well Capex/EUR4 (S/BOE)\n",
|
||||
" 65% 25\n",
|
||||
" Delaware Basin 20\n",
|
||||
" Midland Basin 15\n",
|
||||
" Low HighCost of Supplyz 10 ConocoPhillips\n",
|
||||
" High Single-Digit Production Growth\n",
|
||||
" \"Unconventional acres. 2Source: Enverus and ConocoPhillips (March 2023). 3SourceEnverus (March 2023) based on wells online year: \"Source; Enverus (March 2023). Average single well capex/EUR Top eight public operators based on\n",
|
||||
"wells online in years 2021-2022, greater than 50% oil weight; COP based on COP well design: Competitors include; CVX DVN, EOG; MTDR, OXY, PR and XOM: ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response.source_nodes[7].get_content(metadata_mode=\"all\"))"
|
||||
"print(response.source_nodes[0].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -921,20 +855,26 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"ConocoPhillips has a diverse production base spanning several key geographies:\n",
|
||||
"\n",
|
||||
"1. **Lower 48 (United States)**: This includes major production areas such as the Permian Basin, Bakken Formation, and Eagle Ford Shale.\n",
|
||||
"2. **Alaska**: Significant operations in the North Slope region.\n",
|
||||
"3. **Canada**: Operations in the Montney Formation and the Surmont oil sands project.\n",
|
||||
"4. **EMENA (Europe, Middle East, and North Africa)**: Notable operations in Norway and Libya.\n",
|
||||
"5. **Asia Pacific**: Includes operations in Malaysia, China, and the Australia Pacific LNG (APLNG) project.\n",
|
||||
"6. **Qatar**: Involvement in the country's energy sector.\n",
|
||||
"\n",
|
||||
"These regions highlight the company's extensive and varied geographical footprint in the energy production industry.\n"
|
||||
"Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20/BOE.\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"The average single well capex/EUR for competitors in the Delaware Basin is between $10 and $25 per BOE.\n",
|
||||
"=== LLM Response ===\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# base_response = base_agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
|
||||
"base_response = base_agent.query(\n",
|
||||
" \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
|
||||
")\n",
|
||||
"print(str(base_response))"
|
||||
]
|
||||
},
|
||||
@@ -948,31 +888,30 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Our Differentiated Portfolio: Deep; Durable and Diverse\n",
|
||||
" 20 BBOE of Resource Diverse Production Base\n",
|
||||
" Under $40/BBL Cost of Supply 10-Year Plan Cumulative Production (BBOE)\n",
|
||||
" S50 S32/BBL Lower 48 Alaska\n",
|
||||
" Average Cost of Supply\n",
|
||||
" 3 $40 GKA GWA\n",
|
||||
" GPA WNS\n",
|
||||
" $30 EMENA\n",
|
||||
" 3 Norway\n",
|
||||
" 8 $20\n",
|
||||
" E Qatar Libya\n",
|
||||
" Asia Pacific Canada\n",
|
||||
" $10 Permian\n",
|
||||
" APLNG Montney\n",
|
||||
" S0\n",
|
||||
" 10 15 20 Bakken\n",
|
||||
" Resource (BBOE) Eagle Ford Other Malaysia ChinaSurmont\n",
|
||||
" Lower 48 Canada Alaska EMENA Asia Pacific\n",
|
||||
"Costs assumemid-cycle price environment of S60/BBL WTI:\n",
|
||||
" ConocoPhillips\n"
|
||||
"Deep, Durable and Diverse Portfolio with Significant Growth Runway\n",
|
||||
" 1,2002022 Lower 48 Unconventional Production' (MBOED S50 ~S32/BBL\n",
|
||||
" 000 ConocoPhillips Cost of SupplyAverage\n",
|
||||
" 00 S40\n",
|
||||
" 500 3\n",
|
||||
" 400 1 S30\n",
|
||||
" 200\n",
|
||||
" 5\n",
|
||||
" 15,000ConocoPhillipsNet Remaining Well Inventory? 1 S20\n",
|
||||
" 12,000 S10\n",
|
||||
" 000\n",
|
||||
" 0o0 SO\n",
|
||||
" 3,000 10\n",
|
||||
" Resource (BBOE)\n",
|
||||
" Delaware Basin Midland Basin Eagle Ford Bakken Other\n",
|
||||
" Largest Lower 48 Unconventional Producer; Growing into the Next Decade\n",
|
||||
" onshore operated inventory that achieves 15% IRR at $SO/BBL WTI, Competitors include CVX, DVN, EOG, FANG, MRO, OXY, PXD,and XOM:\n",
|
||||
" Source: Wood Mackenzie Lower 48 Unconventional Plays 2022 ProductionCompetitors include CVX, DVN; EOG, FANG, MRO, OXY, PXD and XOM; greaterthan50% liquids weight: ?Source: Wood Mackenzie (March 2023), Lower 48\n",
|
||||
" ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(base_response.source_nodes[1].get_content(metadata_mode=\"all\"))"
|
||||
"print(base_response.source_nodes[0].get_content(metadata_mode=\"llm\"))"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
File diff suppressed because one or more lines are too long
+1
-3
@@ -4,9 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Building a RAG Pipeline over IKEA Product Instruction Manuals\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/multimodal/product_manual_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
"# Building a RAG Pipeline over IKEA Product Instruction Manuals"
|
||||
]
|
||||
},
|
||||
{
|
||||
+27
-99
@@ -1,22 +1,16 @@
|
||||
import os
|
||||
import asyncio
|
||||
from io import TextIOWrapper
|
||||
|
||||
import httpx
|
||||
import mimetypes
|
||||
import time
|
||||
from pathlib import Path, PurePath, PurePosixPath
|
||||
from typing import AsyncGenerator, Any, Dict, List, Optional, Union
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Union
|
||||
from io import BufferedIOBase
|
||||
|
||||
from fsspec import AbstractFileSystem
|
||||
from fsspec.spec import AbstractBufferedFile
|
||||
from llama_index.core.async_utils import run_jobs
|
||||
from llama_index.core.bridge.pydantic import Field, field_validator
|
||||
from llama_index.core.bridge.pydantic import Field, validator
|
||||
from llama_index.core.constants import DEFAULT_BASE_URL
|
||||
from llama_index.core.readers.base import BasePydanticReader
|
||||
from llama_index.core.readers.file.base import get_default_fs
|
||||
from llama_index.core.schema import Document
|
||||
from llama_parse.utils import (
|
||||
nest_asyncio_err,
|
||||
@@ -37,11 +31,7 @@ _DEFAULT_SEPARATOR = "\n---\n"
|
||||
class LlamaParse(BasePydanticReader):
|
||||
"""A smart-parser for files."""
|
||||
|
||||
api_key: str = Field(
|
||||
default="",
|
||||
description="The API key for the LlamaParse API.",
|
||||
validate_default=True,
|
||||
)
|
||||
api_key: str = Field(default="", description="The API key for the LlamaParse API.")
|
||||
base_url: str = Field(
|
||||
default=DEFAULT_BASE_URL,
|
||||
description="The base URL of the Llama Parsing API.",
|
||||
@@ -91,10 +81,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="Note: Non compatible with gpt-4o. If set to true, the parser will use a faster mode to extract text from documents. This mode will skip OCR of images, and table/heading reconstruction.",
|
||||
)
|
||||
premium_mode: bool = Field(
|
||||
default=False,
|
||||
description="Use our best parser mode if set to True.",
|
||||
)
|
||||
do_not_unroll_columns: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will keep column in the text according to document layout. Reduce reconstruction accuracy, and LLM's/embedings performances in most case.",
|
||||
@@ -147,16 +133,8 @@ class LlamaParse(BasePydanticReader):
|
||||
default=None,
|
||||
description="The model name for the vendor multimodal API.",
|
||||
)
|
||||
take_screenshot: bool = Field(
|
||||
default=False,
|
||||
description="Whether to take screenshot of each page of the document.",
|
||||
)
|
||||
custom_client: Optional[httpx.AsyncClient] = Field(
|
||||
default=None, description="A custom HTTPX client to use for sending requests."
|
||||
)
|
||||
|
||||
@field_validator("api_key", mode="before", check_fields=True)
|
||||
@classmethod
|
||||
@validator("api_key", pre=True, always=True)
|
||||
def validate_api_key(cls, v: str) -> str:
|
||||
"""Validate the API key."""
|
||||
if not v:
|
||||
@@ -169,28 +147,15 @@ class LlamaParse(BasePydanticReader):
|
||||
|
||||
return v
|
||||
|
||||
@field_validator("base_url", mode="before", check_fields=True)
|
||||
@classmethod
|
||||
@validator("base_url", pre=True, always=True)
|
||||
def validate_base_url(cls, v: str) -> str:
|
||||
"""Validate the base URL."""
|
||||
url = os.getenv("LLAMA_CLOUD_BASE_URL", None)
|
||||
return url or v or DEFAULT_BASE_URL
|
||||
|
||||
@asynccontextmanager
|
||||
async def client_context(self) -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""Create a context for the HTTPX client."""
|
||||
if self.custom_client is not None:
|
||||
yield self.custom_client
|
||||
else:
|
||||
async with httpx.AsyncClient(timeout=self.max_timeout) as client:
|
||||
yield client
|
||||
|
||||
# upload a document and get back a job_id
|
||||
async def _create_job(
|
||||
self,
|
||||
file_input: FileInput,
|
||||
extra_info: Optional[dict] = None,
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
self, file_input: FileInput, extra_info: Optional[dict] = None
|
||||
) -> str:
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
url = f"{self.base_url}/api/parsing/upload"
|
||||
@@ -205,7 +170,7 @@ class LlamaParse(BasePydanticReader):
|
||||
file_name = extra_info["file_name"]
|
||||
mime_type = mimetypes.guess_type(file_name)[0]
|
||||
files = {"file": (file_name, file_input, mime_type)}
|
||||
elif isinstance(file_input, (str, Path, PurePosixPath, PurePath)):
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
file_path = str(file_input)
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if file_ext not in SUPPORTED_FILE_TYPES:
|
||||
@@ -215,9 +180,7 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
mime_type = mimetypes.guess_type(file_path)[0]
|
||||
# Open the file here for the duration of the async context
|
||||
# load data, set the mime type
|
||||
fs = fs or get_default_fs()
|
||||
file_handle = fs.open(file_input, "rb")
|
||||
file_handle = open(file_path, "rb")
|
||||
files = {"file": (os.path.basename(file_path), file_handle, mime_type)}
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -231,14 +194,12 @@ class LlamaParse(BasePydanticReader):
|
||||
"skip_diagonal_text": self.skip_diagonal_text,
|
||||
"do_not_cache": self.do_not_cache,
|
||||
"fast_mode": self.fast_mode,
|
||||
"premium_mode": self.premium_mode,
|
||||
"do_not_unroll_columns": self.do_not_unroll_columns,
|
||||
"gpt4o_mode": self.gpt4o_mode,
|
||||
"gpt4o_api_key": self.gpt4o_api_key,
|
||||
"vendor_multimodal_api_key": self.vendor_multimodal_api_key,
|
||||
"use_vendor_multimodal_model": self.use_vendor_multimodal_model,
|
||||
"vendor_multimodal_model_name": self.vendor_multimodal_model_name,
|
||||
"take_screenshot": self.take_screenshot,
|
||||
}
|
||||
|
||||
# only send page separator to server if it is not None
|
||||
@@ -259,7 +220,7 @@ class LlamaParse(BasePydanticReader):
|
||||
data["target_pages"] = self.target_pages
|
||||
|
||||
try:
|
||||
async with self.client_context() as client:
|
||||
async with httpx.AsyncClient(timeout=self.max_timeout) as client:
|
||||
response = await client.post(
|
||||
url,
|
||||
files=files,
|
||||
@@ -274,15 +235,9 @@ class LlamaParse(BasePydanticReader):
|
||||
if file_handle is not None:
|
||||
file_handle.close()
|
||||
|
||||
@staticmethod
|
||||
def __get_filename(f: Union[TextIOWrapper, AbstractBufferedFile]) -> str:
|
||||
if isinstance(f, TextIOWrapper):
|
||||
return f.name
|
||||
return f.full_name
|
||||
|
||||
async def _get_job_result(
|
||||
self, job_id: str, result_type: str, verbose: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict:
|
||||
result_url = f"{self.base_url}/api/parsing/job/{job_id}/result/{result_type}"
|
||||
status_url = f"{self.base_url}/api/parsing/job/{job_id}"
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
@@ -291,7 +246,7 @@ class LlamaParse(BasePydanticReader):
|
||||
tries = 0
|
||||
while True:
|
||||
await asyncio.sleep(self.check_interval)
|
||||
async with self.client_context() as client:
|
||||
async with httpx.AsyncClient(timeout=self.max_timeout) as client:
|
||||
tries += 1
|
||||
|
||||
result = await client.get(status_url, headers=headers)
|
||||
@@ -308,8 +263,7 @@ class LlamaParse(BasePydanticReader):
|
||||
continue
|
||||
|
||||
# Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
|
||||
result_json = result.json()
|
||||
status = result_json["status"]
|
||||
status = result.json()["status"]
|
||||
if status == "SUCCESS":
|
||||
parsed_result = await client.get(result_url, headers=headers)
|
||||
return parsed_result.json()
|
||||
@@ -321,25 +275,22 @@ class LlamaParse(BasePydanticReader):
|
||||
print(".", end="", flush=True)
|
||||
|
||||
await asyncio.sleep(self.check_interval)
|
||||
else:
|
||||
error_code = result_json.get("error_code", "No error code found")
|
||||
error_message = result_json.get(
|
||||
"error_message", "No error message found"
|
||||
)
|
||||
|
||||
exception_str = f"Job ID: {job_id} failed with status: {status}, Error code: {error_code}, Error message: {error_message}"
|
||||
raise Exception(exception_str)
|
||||
continue
|
||||
else:
|
||||
raise Exception(
|
||||
f"Failed to parse the file: {job_id}, status: {status}"
|
||||
)
|
||||
|
||||
async def _aload_data(
|
||||
self,
|
||||
file_path: FileInput,
|
||||
extra_info: Optional[dict] = None,
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
verbose: bool = False,
|
||||
) -> List[Document]:
|
||||
"""Load data from the input path."""
|
||||
try:
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info, fs=fs)
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info)
|
||||
if verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
|
||||
@@ -370,19 +321,17 @@ class LlamaParse(BasePydanticReader):
|
||||
self,
|
||||
file_path: Union[List[FileInput], FileInput],
|
||||
extra_info: Optional[dict] = None,
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
) -> List[Document]:
|
||||
"""Load data from the input path."""
|
||||
if isinstance(file_path, (str, Path, bytes, BufferedIOBase)):
|
||||
return await self._aload_data(
|
||||
file_path, extra_info=extra_info, fs=fs, verbose=self.verbose
|
||||
file_path, extra_info=extra_info, verbose=self.verbose
|
||||
)
|
||||
elif isinstance(file_path, list):
|
||||
jobs = [
|
||||
self._aload_data(
|
||||
f,
|
||||
extra_info=extra_info,
|
||||
fs=fs,
|
||||
verbose=self.verbose and not self.show_progress,
|
||||
)
|
||||
for f in file_path
|
||||
@@ -411,11 +360,10 @@ class LlamaParse(BasePydanticReader):
|
||||
self,
|
||||
file_path: Union[List[FileInput], FileInput],
|
||||
extra_info: Optional[dict] = None,
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
) -> List[Document]:
|
||||
"""Load data from the input path."""
|
||||
try:
|
||||
return asyncio.run(self.aload_data(file_path, extra_info, fs=fs))
|
||||
return asyncio.run(self.aload_data(file_path, extra_info))
|
||||
except RuntimeError as e:
|
||||
if nest_asyncio_err in str(e):
|
||||
raise RuntimeError(nest_asyncio_msg)
|
||||
@@ -430,13 +378,12 @@ class LlamaParse(BasePydanticReader):
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info)
|
||||
if self.verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
|
||||
result = await self._get_job_result(job_id, "json")
|
||||
result["job_id"] = job_id
|
||||
|
||||
if not isinstance(file_path, (bytes, BufferedIOBase)):
|
||||
result["file_path"] = str(file_path)
|
||||
|
||||
result["file_path"] = file_path
|
||||
return [result]
|
||||
|
||||
except Exception as e:
|
||||
file_repr = file_path if isinstance(file_path, str) else "<bytes/buffer>"
|
||||
print(f"Error while parsing the file '{file_repr}':", e)
|
||||
@@ -489,9 +436,7 @@ class LlamaParse(BasePydanticReader):
|
||||
else:
|
||||
raise e
|
||||
|
||||
async def aget_images(
|
||||
self, json_result: List[dict], download_path: str
|
||||
) -> List[dict]:
|
||||
def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
|
||||
"""Download images from the parsed result."""
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
|
||||
@@ -521,18 +466,11 @@ class LlamaParse(BasePydanticReader):
|
||||
|
||||
image["path"] = image_path
|
||||
image["job_id"] = job_id
|
||||
|
||||
image["original_file_path"] = result.get("file_path", None)
|
||||
|
||||
image["original_pdf_path"] = result["file_path"]
|
||||
image["page_number"] = page["page"]
|
||||
with open(image_path, "wb") as f:
|
||||
image_url = f"{self.base_url}/api/parsing/job/{job_id}/result/image/{image_name}"
|
||||
async with self.client_context() as client:
|
||||
res = await client.get(
|
||||
image_url, headers=headers, timeout=self.max_timeout
|
||||
)
|
||||
res.raise_for_status()
|
||||
f.write(res.content)
|
||||
f.write(httpx.get(image_url, headers=headers).content)
|
||||
images.append(image)
|
||||
return images
|
||||
except Exception as e:
|
||||
@@ -542,16 +480,6 @@ class LlamaParse(BasePydanticReader):
|
||||
else:
|
||||
raise e
|
||||
|
||||
def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
|
||||
"""Download images from the parsed result."""
|
||||
try:
|
||||
return asyncio.run(self.aget_images(json_result, download_path))
|
||||
except RuntimeError as e:
|
||||
if nest_asyncio_err in str(e):
|
||||
raise RuntimeError(nest_asyncio_msg)
|
||||
else:
|
||||
raise e
|
||||
|
||||
def _get_sub_docs(self, docs: List[Document]) -> List[Document]:
|
||||
"""Split docs into pages, by separator."""
|
||||
sub_docs = []
|
||||
|
||||
@@ -10,6 +10,7 @@ class ResultType(str, Enum):
|
||||
|
||||
TXT = "text"
|
||||
MD = "markdown"
|
||||
JSON = "json"
|
||||
|
||||
|
||||
class Language(str, Enum):
|
||||
|
||||
Generated
+609
-531
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "llama-parse"
|
||||
version = "0.5.7"
|
||||
version = "0.4.9"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = ["Logan Markewich <logan@llamaindex.ai>"]
|
||||
license = "MIT"
|
||||
@@ -13,7 +13,7 @@ packages = [{include = "llama_parse"}]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.8.1,<4.0"
|
||||
llama-index-core = ">=0.11.0"
|
||||
llama-index-core = ">=0.10.29"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.0.0"
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import os
|
||||
import pytest
|
||||
from fsspec.implementations.local import LocalFileSystem
|
||||
from httpx import AsyncClient
|
||||
|
||||
from llama_parse import LlamaParse
|
||||
|
||||
|
||||
@@ -72,20 +69,6 @@ def test_simple_page_markdown_buffer(markdown_parser: LlamaParse) -> None:
|
||||
assert len(result[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
def test_simple_page_with_custom_fs() -> None:
|
||||
parser = LlamaParse(result_type="markdown")
|
||||
fs = LocalFileSystem()
|
||||
filepath = os.path.join(
|
||||
os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
|
||||
)
|
||||
result = parser.load_data(filepath, fs=fs)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
@@ -110,18 +93,3 @@ def test_simple_page_progress_workers() -> None:
|
||||
result = parser.load_data([filepath, filepath])
|
||||
assert len(result) == 2
|
||||
assert len(result[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
def test_custom_client() -> None:
|
||||
custom_client = AsyncClient(verify=False, timeout=10)
|
||||
parser = LlamaParse(result_type="markdown", custom_client=custom_client)
|
||||
filepath = os.path.join(
|
||||
os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
|
||||
)
|
||||
result = parser.load_data(filepath)
|
||||
assert len(result) == 1
|
||||
assert len(result[0].text) > 0
|
||||
|
||||
Reference in New Issue
Block a user