mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jerry Liu
|
ff5be23813 | ||
|
Jerry Liu
|
6a2e3f3f3e | ||
|
Jerry Liu
|
c92d02805d | ||
|
Jerry Liu
|
7bc5511751 | ||
|
Jerry Liu
|
be65c99274 | ||
|
Jerry Liu
|
2302734e12 | ||
|
Jerry Liu
|
de76b29875 | ||
|
Jerry Liu
|
3d8e4f054a | ||
|
Jerry Liu
|
a8a191ae87 | ||
|
Hemant Malik
|
4d92775aa8 | ||
|
Jerry Liu
|
477847111e | ||
|
Adam Reichert
|
efbcfb1d2e | ||
|
Logan
|
a9b01c761c | ||
|
Pierre-Loic Doulcet
|
dac2f7c84e | ||
|
Jerry Liu
|
18ab095362 | ||
|
Jerry Liu
|
75e83f55a3 | ||
|
Jerry Liu
|
fc23db7d90 | ||
|
Jerry Liu
|
a9c4c6e8b4 | ||
|
Logan Markewich
|
478142e509 | ||
|
Logan Markewich
|
e76d5ba679 | ||
|
Huu Le
|
23dc9c0f68 |
File diff suppressed because one or more lines are too long
@@ -0,0 +1,635 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "97c79c38-38a3-40f3-ba2e-250649347d63",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Multimodal Parsing using Anthropic Claude (Sonnet 3.5)\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/multimodal/claude_parse.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This cookbook shows you how to use LlamaParse to parse any document with the multimodal capabilities of Sonnet 3.5. \n",
|
||||
"\n",
|
||||
"LlamaParse allows you to plug in external, multimodal model vendors for parsing - we handle the error correction, validation, and scalability/reliability for you.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "15e60ecf-519c-41fc-911b-765adaf8bad4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"Download the data. Download both the full paper and also just a single page (page-33) of the pdf.\n",
|
||||
"\n",
|
||||
"Swap in `data/llama2-p33.pdf` for `data/llama2.pdf` in the code blocks below if you want to save on parsing tokens. \n",
|
||||
"\n",
|
||||
"An image of this page is shown below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "91a9e532-1454-40e0-bbf0-fd442c350121",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import nest_asyncio\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0d9fb0aa-74cd-476f-8161-efd9e04248bf",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"--2024-07-11 23:44:38-- https://arxiv.org/pdf/2307.09288\n",
|
||||
"Resolving arxiv.org (arxiv.org)... 151.101.195.42, 151.101.131.42, 151.101.3.42, ...\n",
|
||||
"Connecting to arxiv.org (arxiv.org)|151.101.195.42|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 200 OK\n",
|
||||
"Length: 13661300 (13M) [application/pdf]\n",
|
||||
"Saving to: ‘data/llama2.pdf’\n",
|
||||
"\n",
|
||||
"data/llama2.pdf 100%[===================>] 13.03M 69.3MB/s in 0.2s \n",
|
||||
"\n",
|
||||
"2024-07-11 23:44:38 (69.3 MB/s) - ‘data/llama2.pdf’ saved [13661300/13661300]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!wget \"https://arxiv.org/pdf/2307.09288\" -O data/llama2.pdf\n",
|
||||
"!wget \"https://www.dropbox.com/scl/fi/wpql661uu98vf6e2of2i0/llama2-p33.pdf?rlkey=64weubzkwpmf73y58vbmc8pyi&st=khgx5161&dl=1\" -O data/llama2-p33.pdf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b5c214a2-56fd-4b09-93b3-be994a3b5aa4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4e29a9d7-5bd9-4fb8-8ec1-4c128a748662",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Initialize LlamaParse\n",
|
||||
"\n",
|
||||
"Initialize LlamaParse in multimodal mode, and specify the vendor.\n",
|
||||
"\n",
|
||||
"**NOTE**: optionally you can specify the Anthropic API key. If you do so you will be charged our base LlamaParse price of 0.3c per page. If you don't then you will be charged 6c per page, as we will make the calls to Claude for you."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "dc921729-3446-42ca-8e1b-a6fd26195ed9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core.schema import TextNode\n",
|
||||
"from typing import List\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_text_nodes(json_list: List[dict]):\n",
|
||||
" text_nodes = []\n",
|
||||
" for idx, page in enumerate(json_list):\n",
|
||||
" text_node = TextNode(text=page[\"md\"], metadata={\"page\": page[\"page\"]})\n",
|
||||
" text_nodes.append(text_node)\n",
|
||||
" return text_nodes\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def save_jsonl(data_list, filename):\n",
|
||||
" \"\"\"Save a list of dictionaries as JSON Lines.\"\"\"\n",
|
||||
" with open(filename, \"w\") as file:\n",
|
||||
" for item in data_list:\n",
|
||||
" json.dump(item, file)\n",
|
||||
" file.write(\"\\n\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_jsonl(filename):\n",
|
||||
" \"\"\"Load a list of dictionaries from JSON Lines.\"\"\"\n",
|
||||
" data_list = []\n",
|
||||
" with open(filename, \"r\") as file:\n",
|
||||
" for line in file:\n",
|
||||
" data_list.append(json.loads(line))\n",
|
||||
" return data_list"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f2e9d9cf-8189-4fcb-b34f-cde6cc0b59c8",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 811a29d8-8bcd-4100-bee3-6a83fbde1697\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from llama_parse import LlamaParse\n",
|
||||
"\n",
|
||||
"parser = LlamaParse(\n",
|
||||
" result_type=\"markdown\",\n",
|
||||
" use_vendor_multimodal_model=True,\n",
|
||||
" vendor_multimodal_model_name=\"anthropic-sonnet-3.5\",\n",
|
||||
" # invalidate_cache=True\n",
|
||||
")\n",
|
||||
"json_objs = parser.get_json_result(\"./data/llama2.pdf\")\n",
|
||||
"# json_objs = parser.get_json_result(\"./data/llama2-p33.pdf\")\n",
|
||||
"json_list = json_objs[0][\"pages\"]\n",
|
||||
"docs = get_text_nodes(json_list)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "96a81df0-1026-4e30-a930-f677dc31e344",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Optional: Save\n",
|
||||
"save_jsonl([d.dict() for d in docs], \"docs.jsonl\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ee2e6920-8893-4b39-ae12-94d13c651406",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Optional: Load\n",
|
||||
"from llama_index.core import Document\n",
|
||||
"\n",
|
||||
"docs_dicts = load_jsonl(\"docs.jsonl\")\n",
|
||||
"docs = [Document.parse_obj(d) for d in docs_dicts]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4f3c51b0-7878-48d7-9bc3-02b516500128",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Setup GPT-4o baseline\n",
|
||||
"\n",
|
||||
"For comparison, we will also parse the document using GPT-4o (3c per page)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6fc3f258-50ae-4988-b904-c105463a498f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 04c69ecc-e45d-4ad9-ba72-3045af38268b\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from llama_parse import LlamaParse\n",
|
||||
"\n",
|
||||
"parser_gpt4o = LlamaParse(\n",
|
||||
" result_type=\"markdown\",\n",
|
||||
" use_vendor_multimodal_model=True,\n",
|
||||
" vendor_multimodal_model=\"openai-gpt4o\",\n",
|
||||
" # invalidate_cache=True\n",
|
||||
")\n",
|
||||
"json_objs_gpt4o = parser_gpt4o.get_json_result(\"./data/llama2.pdf\")\n",
|
||||
"# json_objs_gpt4o = parser.get_json_result(\"./data/llama2-p33.pdf\")\n",
|
||||
"json_list_gpt4o = json_objs_gpt4o[0][\"pages\"]\n",
|
||||
"docs_gpt4o = get_text_nodes(json_list_gpt4o)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6a47f04e-12e1-4c80-a71d-ef7721f96401",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Optional: Save\n",
|
||||
"save_jsonl([d.dict() for d in docs_gpt4o], \"docs_gpt4o.jsonl\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c38b5ca3-fa87-434b-b477-bf6a4962eb3d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Optional: Load\n",
|
||||
"from llama_index.core import Document\n",
|
||||
"\n",
|
||||
"docs_gpt4o_dicts = load_jsonl(\"docs_gpt4o.jsonl\")\n",
|
||||
"docs_gpt4o = [Document.parse_obj(d) for d in docs_gpt4o_dicts]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "44c20f7a-2901-4dd0-b635-a4b33c5664c1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## View Results\n",
|
||||
"\n",
|
||||
"Let's visualize the results along with the original document page.\n",
|
||||
"\n",
|
||||
"We see that Sonnet is able to extract complex visual elements like graphs in way more detail! \n",
|
||||
"\n",
|
||||
"**NOTE**: If you're using llama2-p33, just use `docs[0]`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "778698aa-da7e-4081-b3b5-0372f228536f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page: 33\n",
|
||||
"\n",
|
||||
"| Temperature | RLHF v3 | RLHF v2 | RLHF v1 | SFT |\n",
|
||||
"|-------------|---------|---------|---------|-----|\n",
|
||||
"| 0.4 | 98 | 98 | 97 | 95 |\n",
|
||||
"| 0.6 | 97 | 97 | 95 | 94 |\n",
|
||||
"| 0.8 | 97 | 96 | 94 | 92 |\n",
|
||||
"| 1.0 | 96 | 94 | 92 | 89 |\n",
|
||||
"| 1.2 | 95 | 92 | 88 | 83 |\n",
|
||||
"| 1.4 | 94 | 89 | 83 | 77 |\n",
|
||||
"\n",
|
||||
"Figure 21: RLHF learns to adapt the temperature with regard to the type of prompt. Lower Self-BLEU corresponds to more diversity: RLHF eliminates diversity in responses to factual prompts but retains more diversity when generating responses to creative prompts. We prompt each model with a diverse set of 10 creative and 10 factual instructions and sample 25 responses. This is repeated for the temperatures T ∈ {k/10 | k ∈ N : 1 ≤ k ≤ 15}. For each of the 25 responses we compute the Self-BLEU metric and report the mean and standard deviation against the temperature.\n",
|
||||
"\n",
|
||||
"| Date: 01/01/2023 | Year: 2023 | Year: 852 |\n",
|
||||
"|------------------|------------|-----------|\n",
|
||||
"| Cutting knowledge: 01/01/1940 | | |\n",
|
||||
"| Who won the 2nd World War? | Is the Earth flat or round? | Is the Earth flat or round? |\n",
|
||||
"| I'm not sure what you're referring to, as my knowledge stopped in 1940. | The Earth is round. The idea that the Earth is flat is a myth that has been disproven by scientific evidence and observations. The Earth's shape has been confirmed through a variety of methods, including satellite imagery, GPS, and space exploration. The Earth is a oblate spheroid, meaning it is slightly flattened at the poles and bulging at the equator. | In 852, the Earth was believed to be flat by many people. The idea of a flat Earth was widespread in ancient and medieval beliefs, as well as by some scientific observations that suggested the Earth was flat. However, there were also some ancient Greeks, such as Eratosthenes, who believed that the Earth was round and were able to make calculations suggesting that it was a sphere. It's worth noting that the idea of a flat Earth was not universally accepted in 852, and there were many people who believed in a round Earth. Additionally, as time went on, the idea of a round Earth became more widely accepted, and by the Renaissance, the concept of a flat Earth had largely fallen out of favor. |\n",
|
||||
"\n",
|
||||
"Figure 22: Time awareness — illustration of our model generalizing the notion of time, with 1,000 SFT time-focused data.\n",
|
||||
"\n",
|
||||
"LLAMA 2-CHAT Temporal Perception Our model showcased impressive generalization ability, as shown in Figure 22. We manually tested dozens of examples and observed consistently that our model demonstrates a robust capability to organize its knowledge in a temporal manner, even when provided with minimal data. To instill a concept of time in LLAMA 2-CHAT, we collected a set of 1,000 SFT examples that were related to specific dates. These examples included questions like \"How long ago did Barack Obama become president?\" Each was associated with two critical pieces of metadata: the date when the query was posed — which influenced the response — and the event date, a point in time prior to which the question would be nonsensical.\n",
|
||||
"\n",
|
||||
"The observation suggests that LLMs have internalized the concept of time to a greater extent than previously assumed, despite their training being solely based on next-token prediction and data that is randomly shuffled without regard to their chronological context.\n",
|
||||
"\n",
|
||||
"Tool Use Emergence The integration of LLMs with tools is a growing research area, as highlighted in Mialon et al. (2023). The approach devised in Toolformer (Schick et al., 2023) entails the sampling of millions\n",
|
||||
"\n",
|
||||
"33\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# using Sonnet-3.5\n",
|
||||
"print(docs[32].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1511a30f-3efc-4142-9668-7dc056a24d0c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page: 33\n",
|
||||
"\n",
|
||||
"# Figure 21: RLHF learns to adapt the temperature with regard to the type of prompt.\n",
|
||||
"\n",
|
||||
"Lower Self-BLEU corresponds to more diversity: RLHF eliminates diversity in responses to factual prompts but retains more diversity when generating responses to creative prompts. We prompt each model with a diverse set of 10 creative and 10 factual instructions and sample 25 responses. This is repeated for the temperatures \\( T \\in \\{k/10 | k \\in \\{1:1:15\\}\\). For each of the 25 responses we compute the Self-BLEU metric and report the mean and standard deviation against the temperature.\n",
|
||||
"\n",
|
||||
"| Temperature | Factual Prompts | Creative Prompts |\n",
|
||||
"|-------------|-----------------|------------------|\n",
|
||||
"| 0.4 | | |\n",
|
||||
"| 0.6 | | |\n",
|
||||
"| 0.8 | | |\n",
|
||||
"| 1.0 | | |\n",
|
||||
"| 1.2 | | |\n",
|
||||
"| 1.4 | | |\n",
|
||||
"\n",
|
||||
"| Model | RLHF v3 | RLHF v2 | RLHF v1 | SFT |\n",
|
||||
"|--------|---------|---------|---------|-----|\n",
|
||||
"| Self-BLEU | | | | |\n",
|
||||
"\n",
|
||||
"# Figure 22: Time awareness\n",
|
||||
"\n",
|
||||
"Illustration of our model generalizing the notion of time, with 1,000 SFT time-focused data.\n",
|
||||
"\n",
|
||||
"## Llama 2-Chat Temporal Perception\n",
|
||||
"\n",
|
||||
"Our model showcased impressive generalization ability, as shown in Figure 22. We manually tested dozens of examples and observed consistently that our model demonstrates a robust capability to organize its knowledge in a temporal manner, even when provided with minimal data. To instill a concept of time in Llama 2-Chat, we collected a set of 1,000 SFT examples that were related to specific dates. These examples included questions like \"How long ago did Barack Obama become president?\" Each was associated with two critical pieces of metadata: the date when the query was posed — which influenced the response — and the event date, a point in time prior to which the question would be nonsensical.\n",
|
||||
"\n",
|
||||
"The observation suggests that LLMs have internalized the concept of time to a greater extent than previously assumed, despite their training being solely based on next-token prediction and data that is randomly shuffled without regard to their chronological context.\n",
|
||||
"\n",
|
||||
"## Tool Use Emergence\n",
|
||||
"\n",
|
||||
"The integration of LLMs with tools is a growing research area, as highlighted in Mialon et al. (2023). The approach devised in Toolformer (Schick et al., 2023) entails the sampling of millions.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"### Example Prompts and Responses\n",
|
||||
"\n",
|
||||
"| Date: 01/01/2023 | Year: 2023 | Year: 852 |\n",
|
||||
"|------------------|------------|-----------|\n",
|
||||
"| **Who won the 2nd World War?** | **Is the Earth flat or round?** | **Is the Earth flat or round?** |\n",
|
||||
"| I'm not sure what you're referring to, as my knowledge stopped in 1940. | The Earth is round. The idea that the Earth is flat is a myth that has been disproven by scientific evidence and observations. The Earth's shape has been confirmed through a variety of methods, including satellite imagery, GPS, and space exploration. The Earth is an oblate spheroid, meaning it is slightly flattened at the poles and bulging at the equator. | In 852, the Earth was believed to be flat by many people. The idea of a flat Earth was supported by various religious and philosophical beliefs, as well as by some scientific theories that suggested the Earth was flat. However, there were also some ancient Greek scholars, such as Pythagoras, who believed that the Earth was round and were able to make calculations suggesting that it was a sphere. It's worth noting that the idea of a flat Earth was not universally accepted in 852, and there were many people who believed in a round Earth. Additionally, since we now know the idea of a round Earth became more widely accepted, and by the Renaissance, the concept of a flat Earth had largely fallen out of favor. |\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"Page 33\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# using GPT-4o\n",
|
||||
"print(docs_gpt4o[32].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "705f7729-fa0f-4ca0-8562-c42afeaa8532",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup RAG Pipeline\n",
|
||||
"\n",
|
||||
"These parsing capabilities translate to great RAG performance as well. Let's setup a RAG pipeline over this data.\n",
|
||||
"\n",
|
||||
"(we'll use GPT-4o from OpenAI for the actual text synthesis step)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5a53ee5d-cc63-421b-8896-588c83edfcf0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import Settings\n",
|
||||
"from llama_index.llms.openai import OpenAI\n",
|
||||
"from llama_index.embeddings.openai import OpenAIEmbedding\n",
|
||||
"\n",
|
||||
"Settings.llm = OpenAI(model=\"gpt-4o\")\n",
|
||||
"Settings.embed_model = OpenAIEmbedding(model=\"text-embedding-3-large\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "60972d7a-7948-4ad7-89df-57004acee917",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# from llama_index.core import SummaryIndex\n",
|
||||
"from llama_index.core import VectorStoreIndex\n",
|
||||
"from llama_index.llms.openai import OpenAI\n",
|
||||
"\n",
|
||||
"index = VectorStoreIndex(docs)\n",
|
||||
"query_engine = index.as_query_engine(similarity_top_k=5)\n",
|
||||
"\n",
|
||||
"index_gpt4o = VectorStoreIndex(docs_gpt4o)\n",
|
||||
"query_engine_gpt4o = index_gpt4o.as_query_engine(similarity_top_k=5)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e7df7bcb-1df4-4a01-88fc-2d596b1cc74d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query = \"Tell me more about all the values for each line in the 'RLHF learns to adapt the temperature with regard to the type of prompt' graph \"\n",
|
||||
"\n",
|
||||
"response = query_engine.query(query)\n",
|
||||
"response_gpt4o = query_engine_gpt4o.query(query)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b7070a31-3bb8-4134-8338-20bc2fd6f3d6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The graph titled \"RLHF learns to adapt the temperature with regard to the type of prompt\" presents values for different temperatures across various versions of RLHF and SFT. The values are as follows:\n",
|
||||
"\n",
|
||||
"- **Temperature 0.4:**\n",
|
||||
" - RLHF v3: 98\n",
|
||||
" - RLHF v2: 98\n",
|
||||
" - RLHF v1: 97\n",
|
||||
" - SFT: 95\n",
|
||||
"\n",
|
||||
"- **Temperature 0.6:**\n",
|
||||
" - RLHF v3: 97\n",
|
||||
" - RLHF v2: 97\n",
|
||||
" - RLHF v1: 95\n",
|
||||
" - SFT: 94\n",
|
||||
"\n",
|
||||
"- **Temperature 0.8:**\n",
|
||||
" - RLHF v3: 97\n",
|
||||
" - RLHF v2: 96\n",
|
||||
" - RLHF v1: 94\n",
|
||||
" - SFT: 92\n",
|
||||
"\n",
|
||||
"- **Temperature 1.0:**\n",
|
||||
" - RLHF v3: 96\n",
|
||||
" - RLHF v2: 94\n",
|
||||
" - RLHF v1: 92\n",
|
||||
" - SFT: 89\n",
|
||||
"\n",
|
||||
"- **Temperature 1.2:**\n",
|
||||
" - RLHF v3: 95\n",
|
||||
" - RLHF v2: 92\n",
|
||||
" - RLHF v1: 88\n",
|
||||
" - SFT: 83\n",
|
||||
"\n",
|
||||
"- **Temperature 1.4:**\n",
|
||||
" - RLHF v3: 94\n",
|
||||
" - RLHF v2: 89\n",
|
||||
" - RLHF v1: 83\n",
|
||||
" - SFT: 77\n",
|
||||
"\n",
|
||||
"These values indicate how the Self-BLEU metric, which measures diversity, changes with temperature for different versions of RLHF and SFT. Lower Self-BLEU corresponds to more diversity in the responses.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7bee8167-f021-4c87-8d28-9f40a4f7b69d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"| Temperature | RLHF v3 | RLHF v2 | RLHF v1 | SFT |\n",
|
||||
"|-------------|---------|---------|---------|-----|\n",
|
||||
"| 0.4 | 98 | 98 | 97 | 95 |\n",
|
||||
"| 0.6 | 97 | 97 | 95 | 94 |\n",
|
||||
"| 0.8 | 97 | 96 | 94 | 92 |\n",
|
||||
"| 1.0 | 96 | 94 | 92 | 89 |\n",
|
||||
"| 1.2 | 95 | 92 | 88 | 83 |\n",
|
||||
"| 1.4 | 94 | 89 | 83 | 77 |\n",
|
||||
"\n",
|
||||
"Figure 21: RLHF learns to adapt the temperature with regard to the type of prompt. Lower Self-BLEU corresponds to more diversity: RLHF eliminates diversity in responses to factual prompts but retains more diversity when generating responses to creative prompts. We prompt each model with a diverse set of 10 creative and 10 factual instructions and sample 25 responses. This is repeated for the temperatures T ∈ {k/10 | k ∈ N : 1 ≤ k ≤ 15}. For each of the 25 responses we compute the Self-BLEU metric and report the mean and standard deviation against the temperature.\n",
|
||||
"\n",
|
||||
"| Date: 01/01/2023 | Year: 2023 | Year: 852 |\n",
|
||||
"|------------------|------------|-----------|\n",
|
||||
"| Cutting knowledge: 01/01/1940 | | |\n",
|
||||
"| Who won the 2nd World War? | Is the Earth flat or round? | Is the Earth flat or round? |\n",
|
||||
"| I'm not sure what you're referring to, as my knowledge stopped in 1940. | The Earth is round. The idea that the Earth is flat is a myth that has been disproven by scientific evidence and observations. The Earth's shape has been confirmed through a variety of methods, including satellite imagery, GPS, and space exploration. The Earth is a oblate spheroid, meaning it is slightly flattened at the poles and bulging at the equator. | In 852, the Earth was believed to be flat by many people. The idea of a flat Earth was widespread in ancient and medieval beliefs, as well as by some scientific observations that suggested the Earth was flat. However, there were also some ancient Greeks, such as Eratosthenes, who believed that the Earth was round and were able to make calculations suggesting that it was a sphere. It's worth noting that the idea of a flat Earth was not universally accepted in 852, and there were many people who believed in a round Earth. Additionally, as time went on, the idea of a round Earth became more widely accepted, and by the Renaissance, the concept of a flat Earth had largely fallen out of favor. |\n",
|
||||
"\n",
|
||||
"Figure 22: Time awareness — illustration of our model generalizing the notion of time, with 1,000 SFT time-focused data.\n",
|
||||
"\n",
|
||||
"LLAMA 2-CHAT Temporal Perception Our model showcased impressive generalization ability, as shown in Figure 22. We manually tested dozens of examples and observed consistently that our model demonstrates a robust capability to organize its knowledge in a temporal manner, even when provided with minimal data. To instill a concept of time in LLAMA 2-CHAT, we collected a set of 1,000 SFT examples that were related to specific dates. These examples included questions like \"How long ago did Barack Obama become president?\" Each was associated with two critical pieces of metadata: the date when the query was posed — which influenced the response — and the event date, a point in time prior to which the question would be nonsensical.\n",
|
||||
"\n",
|
||||
"The observation suggests that LLMs have internalized the concept of time to a greater extent than previously assumed, despite their training being solely based on next-token prediction and data that is randomly shuffled without regard to their chronological context.\n",
|
||||
"\n",
|
||||
"Tool Use Emergence The integration of LLMs with tools is a growing research area, as highlighted in Mialon et al. (2023). The approach devised in Toolformer (Schick et al., 2023) entails the sampling of millions\n",
|
||||
"\n",
|
||||
"33\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response.source_nodes[4].get_content())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5f9fef7f-510b-46a5-8716-f5616f542035",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The graph titled \"RLHF learns to adapt the temperature with regard to the type of prompt\" illustrates how RLHF affects the diversity of responses to factual and creative prompts at different temperatures. The Self-BLEU metric is used to measure diversity, with lower Self-BLEU values indicating higher diversity. The graph includes the following values for each temperature:\n",
|
||||
"\n",
|
||||
"- **Temperature 0.4**: Values for factual and creative prompts are not provided.\n",
|
||||
"- **Temperature 0.6**: Values for factual and creative prompts are not provided.\n",
|
||||
"- **Temperature 0.8**: Values for factual and creative prompts are not provided.\n",
|
||||
"- **Temperature 1.0**: Values for factual and creative prompts are not provided.\n",
|
||||
"- **Temperature 1.2**: Values for factual and creative prompts are not provided.\n",
|
||||
"- **Temperature 1.4**: Values for factual and creative prompts are not provided.\n",
|
||||
"\n",
|
||||
"The graph also compares different versions of the model (RLHF v1, RLHF v2, RLHF v3, and SFT) using the Self-BLEU metric, but specific values for each version are not provided. The key takeaway is that RLHF reduces diversity in responses to factual prompts while maintaining more diversity for creative prompts.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response_gpt4o)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d40f9dd4-2dd4-4fa5-b636-1f901dc1601b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"# Figure 21: RLHF learns to adapt the temperature with regard to the type of prompt.\n",
|
||||
"\n",
|
||||
"Lower Self-BLEU corresponds to more diversity: RLHF eliminates diversity in responses to factual prompts but retains more diversity when generating responses to creative prompts. We prompt each model with a diverse set of 10 creative and 10 factual instructions and sample 25 responses. This is repeated for the temperatures \\( T \\in \\{k/10 | k \\in \\{1:1:15\\}\\). For each of the 25 responses we compute the Self-BLEU metric and report the mean and standard deviation against the temperature.\n",
|
||||
"\n",
|
||||
"| Temperature | Factual Prompts | Creative Prompts |\n",
|
||||
"|-------------|-----------------|------------------|\n",
|
||||
"| 0.4 | | |\n",
|
||||
"| 0.6 | | |\n",
|
||||
"| 0.8 | | |\n",
|
||||
"| 1.0 | | |\n",
|
||||
"| 1.2 | | |\n",
|
||||
"| 1.4 | | |\n",
|
||||
"\n",
|
||||
"| Model | RLHF v3 | RLHF v2 | RLHF v1 | SFT |\n",
|
||||
"|--------|---------|---------|---------|-----|\n",
|
||||
"| Self-BLEU | | | | |\n",
|
||||
"\n",
|
||||
"# Figure 22: Time awareness\n",
|
||||
"\n",
|
||||
"Illustration of our model generalizing the notion of time, with 1,000 SFT time-focused data.\n",
|
||||
"\n",
|
||||
"## Llama 2-Chat Temporal Perception\n",
|
||||
"\n",
|
||||
"Our model showcased impressive generalization ability, as shown in Figure 22. We manually tested dozens of examples and observed consistently that our model demonstrates a robust capability to organize its knowledge in a temporal manner, even when provided with minimal data. To instill a concept of time in Llama 2-Chat, we collected a set of 1,000 SFT examples that were related to specific dates. These examples included questions like \"How long ago did Barack Obama become president?\" Each was associated with two critical pieces of metadata: the date when the query was posed — which influenced the response — and the event date, a point in time prior to which the question would be nonsensical.\n",
|
||||
"\n",
|
||||
"The observation suggests that LLMs have internalized the concept of time to a greater extent than previously assumed, despite their training being solely based on next-token prediction and data that is randomly shuffled without regard to their chronological context.\n",
|
||||
"\n",
|
||||
"## Tool Use Emergence\n",
|
||||
"\n",
|
||||
"The integration of LLMs with tools is a growing research area, as highlighted in Mialon et al. (2023). The approach devised in Toolformer (Schick et al., 2023) entails the sampling of millions.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"### Example Prompts and Responses\n",
|
||||
"\n",
|
||||
"| Date: 01/01/2023 | Year: 2023 | Year: 852 |\n",
|
||||
"|------------------|------------|-----------|\n",
|
||||
"| **Who won the 2nd World War?** | **Is the Earth flat or round?** | **Is the Earth flat or round?** |\n",
|
||||
"| I'm not sure what you're referring to, as my knowledge stopped in 1940. | The Earth is round. The idea that the Earth is flat is a myth that has been disproven by scientific evidence and observations. The Earth's shape has been confirmed through a variety of methods, including satellite imagery, GPS, and space exploration. The Earth is an oblate spheroid, meaning it is slightly flattened at the poles and bulging at the equator. | In 852, the Earth was believed to be flat by many people. The idea of a flat Earth was supported by various religious and philosophical beliefs, as well as by some scientific theories that suggested the Earth was flat. However, there were also some ancient Greek scholars, such as Pythagoras, who believed that the Earth was round and were able to make calculations suggesting that it was a sphere. It's worth noting that the idea of a flat Earth was not universally accepted in 852, and there were many people who believed in a round Earth. Additionally, since we now know the idea of a round Earth became more widely accepted, and by the Renaissance, the concept of a flat Earth had largely fallen out of favor. |\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"Page 33\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response_gpt4o.source_nodes[4].get_content())"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "llama_parse",
|
||||
"language": "python",
|
||||
"name": "llama_parse"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,938 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93ae9bad-b8cc-43de-ba7d-387e0155674c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Building a Natively Multimodal RAG Pipeline (over a Slide Deck)\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/multimodal/multimodal_rag_slide_deck.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this cookbook we show you how to build a multimodal RAG pipeline over a slide deck, with text, tables, images, diagrams, and complex layouts.\n",
|
||||
"\n",
|
||||
"A gap of text-based RAG is that they struggle with purely text-based representations of complex documents. For instance, if a page contains a lot of images and diagrams, a text parser would need to rely on raw OCR to extract out text. You can also use a multimodal model (e.g. gpt-4o and up) to do text extraction, but this is inherently a lossy conversion.\n",
|
||||
"\n",
|
||||
"Instead a **native multimodal pipeline** stores both a text and image representation of a document chunk. They are indexed via embeddings (text or image), and during synthesis both text and image are directly fed to the multimodal model for synthesis.\n",
|
||||
"\n",
|
||||
"This can have the following advantages:\n",
|
||||
"- **Robustness**: This solution is more robust than a pure text or even a pure image-based approach. In a pure text RAG approach, the parsing piece can be lossy. In a pure image-based approach, multimodal OCR is not perfect and may lose out against text parsing for text-heavy documents.\n",
|
||||
"- **Cost Optimization**: You may choose to dynamically include text-only, or text + image depending on the content of the page.\n",
|
||||
"\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54e8d9a7-5036-4d32-818f-00b2e888521f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70ccdd53-e68a-4199-aacb-cfe71ad1ff0b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import nest_asyncio\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "225c5556-a789-4386-a1ee-cce01dbeb6cf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Setup Observability\n",
|
||||
"\n",
|
||||
"We setup an integration with LlamaTrace (integration with Arize).\n",
|
||||
"\n",
|
||||
"If you haven't already done so, make sure to create an account here: https://llamatrace.com/login. Then create an API key and put it in the `PHOENIX_API_KEY` variable below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0eabee1f-290a-4c85-b362-54f45c8559ae",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -U llama-index-callbacks-arize-phoenix"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aaeb245c-730b-4c34-ad68-708fdde0e6cb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# setup Arize Phoenix for logging/observability\n",
|
||||
"import llama_index.core\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"PHOENIX_API_KEY = \"<PHOENIX_API_KEY>\"\n",
|
||||
"os.environ[\"OTEL_EXPORTER_OTLP_HEADERS\"] = f\"api_key={PHOENIX_API_KEY}\"\n",
|
||||
"llama_index.core.set_global_handler(\n",
|
||||
" \"arize_phoenix\", endpoint=\"https://llamatrace.com/v1/traces\"\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fbb362db-b1b1-4eea-be1a-b1f78b0779d7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Load Data\n",
|
||||
"\n",
|
||||
"Here we load the [Conoco Phillips 2023 investor meeting slide deck](https://static.conocophillips.com/files/2023-conocophillips-aim-presentation.pdf)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8bce3407-a7d2-47e8-9eaf-ab297a94750c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!mkdir data\n",
|
||||
"!mkdir data_images\n",
|
||||
"!wget \"https://static.conocophillips.com/files/2023-conocophillips-aim-presentation.pdf\" -O data/conocophillips.pdf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "246ba6b0-51af-42f9-b1b2-8d3e721ef782",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Model Setup\n",
|
||||
"\n",
|
||||
"Setup models that will be used for downstream orchestration."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "16e2071d-bbc2-4707-8ae7-cb4e1fecafd3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import Settings\n",
|
||||
"from llama_index.llms.openai import OpenAI\n",
|
||||
"from llama_index.embeddings.openai import OpenAIEmbedding\n",
|
||||
"\n",
|
||||
"embed_model = OpenAIEmbedding(model=\"text-embedding-3-large\")\n",
|
||||
"llm = OpenAI(model=\"gpt-4o\")\n",
|
||||
"\n",
|
||||
"Settings.embed_model = embed_model\n",
|
||||
"Settings.llm = llm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e3f6416f-f580-4722-aaa9-7f3500408547",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Use LlamaParse to Parse Text and Images\n",
|
||||
"\n",
|
||||
"In this example, use LlamaParse to parse both the text and images from the document.\n",
|
||||
"\n",
|
||||
"We parse out the text in two ways: \n",
|
||||
"- in regular `text` mode using our default text layout algorithm\n",
|
||||
"- in `markdown` mode using GPT-4o (`gpt4o_mode=True`). This also allows us to capture page screenshots"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "570089e5-238a-4dcc-af65-96e7393c2b4d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_parse import LlamaParse\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"parser_text = LlamaParse(result_type=\"text\")\n",
|
||||
"parser_gpt4o = LlamaParse(result_type=\"markdown\", gpt4o_mode=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ef82a985-4088-4bb7-9a21-0318e1b9207d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(f\"Parsing text...\")\n",
|
||||
"docs_text = parser_text.load_data(\"data/conocophillips.pdf\")\n",
|
||||
"print(f\"Parsing PDF file...\")\n",
|
||||
"md_json_objs = parser_gpt4o.get_json_result(\"data/conocophillips.pdf\")\n",
|
||||
"md_json_list = md_json_objs[0][\"pages\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7506b603-c01f-45de-b354-4a0728dde03c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(docs_text[0].get_content())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5318fb7b-fe6a-4a8a-b82e-4ed7b4512c37",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(md_json_list[10][\"md\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7a46a73e-c6e2-4b0b-bd10-31b0d3e4b70f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"dict_keys(['page', 'text', 'md', 'images', 'items'])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(md_json_list[1].keys())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "eeadb16c-97eb-4622-9551-b34d7f90d72f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"image_dicts = parser_gpt4o.get_images(md_json_objs, download_path=\"data_images\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fd3e098b-0606-4429-b48d-d4fe0140fc0e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Build Multimodal Index\n",
|
||||
"\n",
|
||||
"In this section we build the multimodal index over the parsed deck. \n",
|
||||
"\n",
|
||||
"We do this by creating **text** nodes from the document that contain metadata referencing the original image path.\n",
|
||||
"\n",
|
||||
"In this example we're indexing the text node for retrieval. The text node has a reference to both the parsed text as well as the image screenshot."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3aae2dee-9d85-4604-8a51-705d4db527f7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Get Text Nodes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "18c24174-05ce-417f-8dd2-79c3f375db03",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core.schema import TextNode\n",
|
||||
"from typing import Optional"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8e331dfe-a627-4e23-8c57-70ab1d9342e4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get pages loaded through llamaparse\n",
|
||||
"import re\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_page_number(file_name):\n",
|
||||
" match = re.search(r\"-page-(\\d+)\\.jpg$\", str(file_name))\n",
|
||||
" if match:\n",
|
||||
" return int(match.group(1))\n",
|
||||
" return 0\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _get_sorted_image_files(image_dir):\n",
|
||||
" \"\"\"Get image files sorted by page.\"\"\"\n",
|
||||
" raw_files = [f for f in list(Path(image_dir).iterdir()) if f.is_file()]\n",
|
||||
" sorted_files = sorted(raw_files, key=get_page_number)\n",
|
||||
" return sorted_files"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "346fe5ef-171e-4a54-9084-7a7805103a13",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from copy import deepcopy\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# attach image metadata to the text nodes\n",
|
||||
"def get_text_nodes(docs, image_dir=None, json_dicts=None):\n",
|
||||
" \"\"\"Split docs into nodes, by separator.\"\"\"\n",
|
||||
" nodes = []\n",
|
||||
"\n",
|
||||
" image_files = _get_sorted_image_files(image_dir) if image_dir is not None else None\n",
|
||||
" md_texts = [d[\"md\"] for d in json_dicts] if json_dicts is not None else None\n",
|
||||
"\n",
|
||||
" doc_chunks = docs[0].text.split(\"---\")\n",
|
||||
" for idx, doc_chunk in enumerate(doc_chunks):\n",
|
||||
" chunk_metadata = {\"page_num\": idx + 1}\n",
|
||||
" if image_files is not None:\n",
|
||||
" image_file = image_files[idx]\n",
|
||||
" chunk_metadata[\"image_path\"] = str(image_file)\n",
|
||||
" if md_texts is not None:\n",
|
||||
" chunk_metadata[\"parsed_text_markdown\"] = md_texts[idx]\n",
|
||||
" chunk_metadata[\"parsed_text\"] = doc_chunk\n",
|
||||
" node = TextNode(\n",
|
||||
" text=\"\",\n",
|
||||
" metadata=chunk_metadata,\n",
|
||||
" )\n",
|
||||
" nodes.append(node)\n",
|
||||
"\n",
|
||||
" return nodes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f591669c-5a8e-491d-9cef-0b754abbf26f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# this will split into pages\n",
|
||||
"text_nodes = get_text_nodes(docs_text, image_dir=\"data_images\", json_dicts=md_json_list)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "32c13950-c1db-435f-b5b4-89d62b8b7744",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page_num: 11\n",
|
||||
"image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-10.jpg\n",
|
||||
"parsed_text_markdown: # Commitment to Disciplined Reinvestment Rate\n",
|
||||
"\n",
|
||||
"| Year | Reinvestment Rate | WTI Average Price | Reinvestment Rate at $60/BBL WTI | Reinvestment Rate at $80/BBL WTI |\n",
|
||||
"|------------|-------------------|-------------------|----------------------------------|----------------------------------|\n",
|
||||
"| 2012-2016 | >100% | ~$75/BBL | | |\n",
|
||||
"| 2017-2022 | <60% | ~$63/BBL | | |\n",
|
||||
"| 2023E | | | | at $80/BBL WTI |\n",
|
||||
"| 2024-2028 | | | at $60/BBL WTI | at $80/BBL WTI |\n",
|
||||
"| 2029-2032 | | | at $60/BBL WTI | at $80/BBL WTI |\n",
|
||||
"\n",
|
||||
"**Disciplined Reinvestment Rate is the Foundation for Superior Returns on and of Capital, while Driving Durable CFO Growth**\n",
|
||||
"\n",
|
||||
"- ~50% 10-Year Reinvestment Rate\n",
|
||||
"- ~6% CFO CAGR 2024-2032 at $60/BBL WTI Mid-Cycle Planning Price\n",
|
||||
"\n",
|
||||
"**Note:** Reinvestment rate and cash from operations (CFO) are non-GAAP measures. Definitions and reconciliations are included in the Appendix.\n",
|
||||
"parsed_text: \n",
|
||||
"Commitment to Disciplined Reinvestment Rate\n",
|
||||
" Industry ConocoPhillips\n",
|
||||
" Strategy Reset Disciplined Reinvestment Rate is the Foundation for Superior\n",
|
||||
" Growth Focus Returns on and of Capital, while Driving Durable CFO Growth\n",
|
||||
" 100% <60% 50% 6% at $60/BBL WTI\n",
|
||||
" Reinvestment Rate Reinvestment Rate Reinvestment Rate10-YearCFO CAGR Planning PriceMid-Cycle\n",
|
||||
" 2024-2032\n",
|
||||
" 2 100%\n",
|
||||
" 1 75%\n",
|
||||
" 1 50%\n",
|
||||
" 1 WTIat $80/BBL at S80/BBL\n",
|
||||
" 25% 'S75/BBL $63/BBL WTI\n",
|
||||
" WTI WTI at S80/BBL at S60/BBL at S60/BBL\n",
|
||||
" Average Average WTI WTI WTI\n",
|
||||
" 0%\n",
|
||||
" 2012-2016 2017-2022 2023E 2024-2028 2029-2032\n",
|
||||
" Historic Reinvestment Rate Reinvestment Rate at $60/BBL WTI Reinvestment Rate at $80/BBL WTI\n",
|
||||
" Reinvestment rate andcashfrom operations (CFO) are non-GAAP measures: Definitions and reconciliations are included in the Appendix ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(text_nodes[10].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4f404f56-db1e-4ed7-9ba1-ead763546348",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Build Index\n",
|
||||
"\n",
|
||||
"Once the text nodes are ready, we feed into our vector store index abstraction, which will index these nodes into a simple in-memory vector store (of course, you should definitely check out our 40+ vector store integrations!)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6ea53c31-0e38-421c-8d9b-0e3adaa1677e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from llama_index.core import (\n",
|
||||
" StorageContext,\n",
|
||||
" VectorStoreIndex,\n",
|
||||
" load_index_from_storage,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if not os.path.exists(\"storage_nodes\"):\n",
|
||||
" index = VectorStoreIndex(text_nodes, embed_model=embed_model)\n",
|
||||
" # save index to disk\n",
|
||||
" index.set_index_id(\"vector_index\")\n",
|
||||
" index.storage_context.persist(\"./storage_nodes\")\n",
|
||||
"else:\n",
|
||||
" # rebuild storage context\n",
|
||||
" storage_context = StorageContext.from_defaults(persist_dir=\"storage_nodes\")\n",
|
||||
" # load index\n",
|
||||
" index = load_index_from_storage(storage_context, index_id=\"vector_index\")\n",
|
||||
"\n",
|
||||
"retriever = index.as_retriever()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5f0e33a4-9422-498d-87ee-d917bdf74d80",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Build Multimodal Query Engine\n",
|
||||
"\n",
|
||||
"We now use LlamaIndex abstractions to build a **custom query engine**. In contrast to a standard RAG query engine that will retrieve the text node and only put that into the prompt (response synthesis module), this custom query engine will also load the image document, and put both the text and image document into the response synthesis module."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "35a94be2-e289-41a6-92e4-d3cb428fb0c8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core.query_engine import CustomQueryEngine, SimpleMultiModalQueryEngine\n",
|
||||
"from llama_index.core.retrievers import BaseRetriever\n",
|
||||
"from llama_index.multi_modal_llms.openai import OpenAIMultiModal\n",
|
||||
"from llama_index.core.schema import ImageNode, NodeWithScore, MetadataMode\n",
|
||||
"from llama_index.core.prompts import PromptTemplate\n",
|
||||
"from llama_index.core.base.response.schema import Response\n",
|
||||
"from typing import Optional\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"gpt_4o = OpenAIMultiModal(model=\"gpt-4o\", max_new_tokens=4096)\n",
|
||||
"\n",
|
||||
"QA_PROMPT_TMPL = \"\"\"\\\n",
|
||||
"Below we give parsed text from slides in two different formats, as well as the image.\n",
|
||||
"\n",
|
||||
"We parse the text in both 'markdown' mode as well as 'raw text' mode. Markdown mode attempts \\\n",
|
||||
"to convert relevant diagrams into tables, whereas raw text tries to maintain the rough spatial \\\n",
|
||||
"layout of the text.\n",
|
||||
"\n",
|
||||
"Use the image information first and foremost. ONLY use the text/markdown information \n",
|
||||
"if you can't understand the image.\n",
|
||||
"\n",
|
||||
"---------------------\n",
|
||||
"{context_str}\n",
|
||||
"---------------------\n",
|
||||
"Given the context information and not prior knowledge, answer the query. Explain whether you got the answer\n",
|
||||
"from the parsed markdown or raw text or image, and if there's discrepancies, and your reasoning for the final answer.\n",
|
||||
"\n",
|
||||
"Query: {query_str}\n",
|
||||
"Answer: \"\"\"\n",
|
||||
"\n",
|
||||
"QA_PROMPT = PromptTemplate(QA_PROMPT_TMPL)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class MultimodalQueryEngine(CustomQueryEngine):\n",
|
||||
" \"\"\"Custom multimodal Query Engine.\n",
|
||||
"\n",
|
||||
" Takes in a retriever to retrieve a set of document nodes.\n",
|
||||
" Also takes in a prompt template and multimodal model.\n",
|
||||
"\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" qa_prompt: PromptTemplate\n",
|
||||
" retriever: BaseRetriever\n",
|
||||
" multi_modal_llm: OpenAIMultiModal\n",
|
||||
"\n",
|
||||
" def __init__(self, qa_prompt: Optional[PromptTemplate] = None, **kwargs) -> None:\n",
|
||||
" \"\"\"Initialize.\"\"\"\n",
|
||||
" super().__init__(qa_prompt=qa_prompt or QA_PROMPT, **kwargs)\n",
|
||||
"\n",
|
||||
" def custom_query(self, query_str: str):\n",
|
||||
" # retrieve text nodes\n",
|
||||
" nodes = self.retriever.retrieve(query_str)\n",
|
||||
" # create ImageNode items from text nodes\n",
|
||||
" image_nodes = [\n",
|
||||
" NodeWithScore(node=ImageNode(image_path=n.metadata[\"image_path\"]))\n",
|
||||
" for n in nodes\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" # create context string from text nodes, dump into the prompt\n",
|
||||
" context_str = \"\\n\\n\".join(\n",
|
||||
" [r.get_content(metadata_mode=MetadataMode.LLM) for r in nodes]\n",
|
||||
" )\n",
|
||||
" fmt_prompt = self.qa_prompt.format(context_str=context_str, query_str=query_str)\n",
|
||||
"\n",
|
||||
" # synthesize an answer from formatted text and images\n",
|
||||
" llm_response = self.multi_modal_llm.complete(\n",
|
||||
" prompt=fmt_prompt,\n",
|
||||
" image_documents=[image_node.node for image_node in image_nodes],\n",
|
||||
" )\n",
|
||||
" return Response(\n",
|
||||
" response=str(llm_response),\n",
|
||||
" source_nodes=nodes,\n",
|
||||
" metadata={\"text_nodes\": text_nodes, \"image_nodes\": image_nodes},\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" return response"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0890be59-fb12-4bb5-959b-b2d9600f7774",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_engine = MultimodalQueryEngine(\n",
|
||||
" retriever=index.as_retriever(similarity_top_k=9), multi_modal_llm=gpt_4o\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a92aa4f1-7501-4711-b054-f02338e54e74",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define Baseline\n",
|
||||
"\n",
|
||||
"In addition, we define a \"baseline\" where we rely only on text-based indexing. Here we define an index using only the nodes that are parsed in text-mode from LlamaParse. \n",
|
||||
"\n",
|
||||
"**NOTE**: We don't currently include the markdown-parsed text because that was parsed with GPT-4o, so already uses a multimodal model during the text extraction phase.\n",
|
||||
"\n",
|
||||
"It is of course a valid experiment to compare RAG where multimodal extraction only happens during indexing, vs. the current multimodal RAG implementation where images are fed during synthesis to the LLM. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c0b15a48-d177-4666-aec2-98ee90664642",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_nodes(docs):\n",
|
||||
" \"\"\"Split docs into nodes, by separator.\"\"\"\n",
|
||||
" nodes = []\n",
|
||||
" for doc in docs:\n",
|
||||
" doc_chunks = doc.text.split(\"\\n---\\n\")\n",
|
||||
" for doc_chunk in doc_chunks:\n",
|
||||
" node = TextNode(\n",
|
||||
" text=doc_chunk,\n",
|
||||
" metadata=deepcopy(doc.metadata),\n",
|
||||
" )\n",
|
||||
" nodes.append(node)\n",
|
||||
"\n",
|
||||
" return nodes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2065d2c6-d6ba-4ee3-8e9e-dbc83cbcec1b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"base_nodes = get_nodes(docs_text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bcaea1a8-26c9-4385-8f62-32855aa898b6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Our Differentiated Portfolio: Deep; Durable and Diverse\n",
|
||||
" 20 BBOE of Resource Diverse Production Base\n",
|
||||
" Under $40/BBL Cost of Supply 10-Year Plan Cumulative Production (BBOE)\n",
|
||||
" S50 S32/BBL Lower 48 Alaska\n",
|
||||
" Average Cost of Supply\n",
|
||||
" 3$40 GKA GWA\n",
|
||||
" GPA WNS\n",
|
||||
" $30 EMENA\n",
|
||||
" 3 Norway\n",
|
||||
" 8 $20\n",
|
||||
" E Qatar Libya\n",
|
||||
" Asia Pacific Canada\n",
|
||||
" $10 Permian\n",
|
||||
" APLNG Montney\n",
|
||||
" S0\n",
|
||||
" 10 15 20 Bakken\n",
|
||||
" Resource (BBOE) Eagle Ford Other MalaysiaChina Surmont\n",
|
||||
" Lower 48 Canada Alaska EMENA Asia Pacific\n",
|
||||
"Costs assumemid-cycle price environment of S60/BBL WTI:\n",
|
||||
" ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(base_nodes[13].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f6bcfbc6-4e9b-41ad-ad81-1c4245b95cd5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"base_index = VectorStoreIndex(base_nodes, embed_model=embed_model)\n",
|
||||
"base_query_engine = base_index.as_query_engine(llm=llm, similarity_top_k=9)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1f94ef26-0df5-4468-a156-903d686f02ce",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Build a Multimodal Agent\n",
|
||||
"\n",
|
||||
"Build an agent around the multimodal query engine. This gives you agent capabilities like query planning/decomposition and memory around a central QA interface."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5b7a8c5f-39fc-4d04-8c56-3642f5718437",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core.tools import QueryEngineTool\n",
|
||||
"from llama_index.core.agent import FunctionCallingAgentWorker\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"vector_tool = QueryEngineTool.from_defaults(\n",
|
||||
" query_engine=query_engine,\n",
|
||||
" name=\"vector_tool\",\n",
|
||||
" description=(\n",
|
||||
" \"Useful for retrieving specific context from the data. Do NOT select if question asks for a summary of the data.\"\n",
|
||||
" ),\n",
|
||||
")\n",
|
||||
"agent = FunctionCallingAgentWorker.from_tools(\n",
|
||||
" [vector_tool], llm=llm, verbose=True\n",
|
||||
").as_agent()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2b4f7eb1-d247-45fa-bb41-c02fc353a22a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# define a similar agent for the baseline\n",
|
||||
"base_vector_tool = QueryEngineTool.from_defaults(\n",
|
||||
" query_engine=base_query_engine,\n",
|
||||
" name=\"vector_tool\",\n",
|
||||
" description=(\n",
|
||||
" \"Useful for retrieving specific context from the data. Do NOT select if question asks for a summary of the data.\"\n",
|
||||
" ),\n",
|
||||
")\n",
|
||||
"base_agent = FunctionCallingAgentWorker.from_tools(\n",
|
||||
" [base_vector_tool], llm=llm, verbose=True\n",
|
||||
").as_agent()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2336f98b-c0a1-413a-849d-8a89bacb90b5",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Try out Queries\n",
|
||||
"\n",
|
||||
"Let's try out queries against these documents and compare against each other."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1cdce5d8-6bb3-4cd3-929d-1cec249d9052",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"The ConocoPhillips capex/EUR in the Delaware Basin is $10/BOE.\n",
|
||||
"\n",
|
||||
"I obtained this information from the image provided. The image clearly shows a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\" where ConocoPhillips is listed with a capex/EUR of $10/BOE. This information is consistent with the parsed markdown text, which also lists ConocoPhillips' capex/EUR as $10/BOE in the Delaware Basin. There are no discrepancies between the image and the parsed markdown text in this case.\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"The competitors' Capex/EUR in the Delaware Basin can be found in the image on the slide titled \"Delaware: Vast Inventory with Proven Track Record of Performance.\" The relevant information is presented in a bar chart under the section \"Delaware Basin Well Capex/EUR ($/BOE)\".\n",
|
||||
"\n",
|
||||
"Here are the details:\n",
|
||||
"\n",
|
||||
"- ConocoPhillips: $10/BOE\n",
|
||||
"- Competitor 1: $15/BOE\n",
|
||||
"- Competitor 2: $20/BOE\n",
|
||||
"- Competitor 3: $25/BOE\n",
|
||||
"- Competitor 4: $30/BOE\n",
|
||||
"- Competitor 5: $35/BOE\n",
|
||||
"- Competitor 6: $40/BOE\n",
|
||||
"- Competitor 7: $45/BOE\n",
|
||||
"\n",
|
||||
"This information was obtained directly from the image, which provides a clear visual representation of the Capex/EUR values for ConocoPhillips and its competitors in the Delaware Basin. The parsed markdown text also confirms these values, ensuring consistency between the image and the text.\n",
|
||||
"=== LLM Response ===\n",
|
||||
"The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
|
||||
"\n",
|
||||
"- **ConocoPhillips**: $10/BOE\n",
|
||||
"- **Competitor 1**: $15/BOE\n",
|
||||
"- **Competitor 2**: $20/BOE\n",
|
||||
"- **Competitor 3**: $25/BOE\n",
|
||||
"- **Competitor 4**: $30/BOE\n",
|
||||
"- **Competitor 5**: $35/BOE\n",
|
||||
"- **Competitor 6**: $40/BOE\n",
|
||||
"- **Competitor 7**: $45/BOE\n",
|
||||
"\n",
|
||||
"This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n",
|
||||
"The capital expenditure per estimated ultimate recovery (capex/EUR) for ConocoPhillips in the Delaware Basin is $10 per barrel of oil equivalent (BOE). When compared to its competitors, ConocoPhillips has a significantly lower capex/EUR. Here are the capex/EUR values for ConocoPhillips and its competitors:\n",
|
||||
"\n",
|
||||
"- **ConocoPhillips**: $10/BOE\n",
|
||||
"- **Competitor 1**: $15/BOE\n",
|
||||
"- **Competitor 2**: $20/BOE\n",
|
||||
"- **Competitor 3**: $25/BOE\n",
|
||||
"- **Competitor 4**: $30/BOE\n",
|
||||
"- **Competitor 5**: $35/BOE\n",
|
||||
"- **Competitor 6**: $40/BOE\n",
|
||||
"- **Competitor 7**: $45/BOE\n",
|
||||
"\n",
|
||||
"This data indicates that ConocoPhillips has a more cost-efficient operation in the Delaware Basin compared to its competitors.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# response = agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
|
||||
"response = agent.query(\n",
|
||||
" \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
|
||||
")\n",
|
||||
"print(str(response))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d584c560-8f49-4c10-a4db-2e0d3b7085d2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"page_num: 38\n",
|
||||
"image_path: data_images/d9137e19-3974-4b5d-998f-dac0cf29dd9d-page-37.jpg\n",
|
||||
"parsed_text_markdown: # Delaware: Vast Inventory with Proven Track Record of Performance\n",
|
||||
"\n",
|
||||
"## Prolific Acreage Spanning Over ~659,000 Net Acres¹\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Total 10-Year Operated Permian Inventory\n",
|
||||
"\n",
|
||||
"- Delaware Basin: 65%\n",
|
||||
"- Midland Basin: 35%\n",
|
||||
"\n",
|
||||
"### High Single-Digit Production Growth\n",
|
||||
"\n",
|
||||
"## 12-Month Cumulative Production³ (BOE/FT)\n",
|
||||
"\n",
|
||||
"| Months | 2019 | 2020 | 2021 | 2022 |\n",
|
||||
"|--------|------|------|------|------|\n",
|
||||
"| 1 | 0 | 0 | 0 | 0 |\n",
|
||||
"| 2 | 5 | 6 | 7 | 8 |\n",
|
||||
"| 3 | 10 | 12 | 14 | 16 |\n",
|
||||
"| 4 | 15 | 18 | 21 | 24 |\n",
|
||||
"| 5 | 20 | 24 | 28 | 32 |\n",
|
||||
"| 6 | 25 | 30 | 35 | 40 |\n",
|
||||
"| 7 | 30 | 36 | 42 | 48 |\n",
|
||||
"| 8 | 35 | 42 | 49 | 56 |\n",
|
||||
"| 9 | 40 | 48 | 56 | 64 |\n",
|
||||
"| 10 | 45 | 54 | 63 | 72 |\n",
|
||||
"| 11 | 50 | 60 | 70 | 80 |\n",
|
||||
"| 12 | 55 | 66 | 77 | 88 |\n",
|
||||
"\n",
|
||||
"~30% Improved Performance from 2019 to 2022\n",
|
||||
"\n",
|
||||
"## Delaware Basin Well Capex/EUR⁴ ($/BOE)\n",
|
||||
"\n",
|
||||
"| Company | Capex/EUR |\n",
|
||||
"|------------------|-----------|\n",
|
||||
"| ConocoPhillips | 10 |\n",
|
||||
"| Competitor 1 | 15 |\n",
|
||||
"| Competitor 2 | 20 |\n",
|
||||
"| Competitor 3 | 25 |\n",
|
||||
"| Competitor 4 | 30 |\n",
|
||||
"| Competitor 5 | 35 |\n",
|
||||
"| Competitor 6 | 40 |\n",
|
||||
"| Competitor 7 | 45 |\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"¹ Unconventional acres. \n",
|
||||
"² Source: Enverus and ConocoPhillips (March 2023). \n",
|
||||
"³ Source: Enverus (March 2023) based on wells online year. \n",
|
||||
"⁴ Source: Enverus (March 2023). Average single well capex/EUR. Top eight public operators based on wells online in years 2021-2022, greater than 50% oil weight. COP based on COP well design. Competitors include: CVX, DVN, EOG, MTDR, OXY, PR and XOM.\n",
|
||||
"parsed_text: \n",
|
||||
"Delaware: Vast Inventory with Proven Track Record of Performance\n",
|
||||
" New Prolific Acreage Spanning Over 12-Month Cumulative Production? (BOE/FT)\n",
|
||||
" Mexico 659,000 Net Acres' 40\n",
|
||||
" Texas 3828\n",
|
||||
" 30 2019\n",
|
||||
" 20 30%\n",
|
||||
" 10 Improved Performancefrom 2019 to 2022\n",
|
||||
" Total\n",
|
||||
" Permian Inventory\n",
|
||||
" 10-Year Operated\n",
|
||||
" 2 10 11 12\n",
|
||||
" Months\n",
|
||||
" Delaware Basin Well Capex/EUR4 (S/BOE)\n",
|
||||
" 65% 25\n",
|
||||
" Delaware Basin 20\n",
|
||||
" Midland Basin 15\n",
|
||||
" Low HighCost of Supplyz 10 ConocoPhillips\n",
|
||||
" High Single-Digit Production Growth\n",
|
||||
" \"Unconventional acres. 2Source: Enverus and ConocoPhillips (March 2023). 3SourceEnverus (March 2023) based on wells online year: \"Source; Enverus (March 2023). Average single well capex/EUR Top eight public operators based on\n",
|
||||
"wells online in years 2021-2022, greater than 50% oil weight; COP based on COP well design: Competitors include; CVX DVN, EOG; MTDR, OXY, PR and XOM: ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(response.source_nodes[0].get_content(metadata_mode=\"all\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d21d694b-6618-4d04-a6f6-8b0c2625f539",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Added user message to memory: How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"Conoco Phillips capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20/BOE.\n",
|
||||
"=== Calling Function ===\n",
|
||||
"Calling function: vector_tool with args: {\"input\": \"competitors capex/EUR in the Delaware Basin\"}\n",
|
||||
"=== Function Output ===\n",
|
||||
"The average single well capex/EUR for competitors in the Delaware Basin is between $10 and $25 per BOE.\n",
|
||||
"=== LLM Response ===\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n",
|
||||
"ConocoPhillips' capex/EUR in the Delaware Basin is approximately $20 per BOE. In comparison, the average capex/EUR for competitors in the Delaware Basin ranges between $10 and $25 per BOE. This places ConocoPhillips' capex/EUR towards the higher end of the competitive range.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# base_response = base_agent.query(\"Tell me about the different regions and subregions where Conoco Phillips has a production base.\")\n",
|
||||
"base_response = base_agent.query(\n",
|
||||
" \"How does the Conoco Phillips capex/EUR in the delaware basin compare against other competitors?\"\n",
|
||||
")\n",
|
||||
"print(str(base_response))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d3afccae-ad8d-4c5d-9d93-810dba413a5d",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Deep, Durable and Diverse Portfolio with Significant Growth Runway\n",
|
||||
" 1,2002022 Lower 48 Unconventional Production' (MBOED S50 ~S32/BBL\n",
|
||||
" 000 ConocoPhillips Cost of SupplyAverage\n",
|
||||
" 00 S40\n",
|
||||
" 500 3\n",
|
||||
" 400 1 S30\n",
|
||||
" 200\n",
|
||||
" 5\n",
|
||||
" 15,000ConocoPhillipsNet Remaining Well Inventory? 1 S20\n",
|
||||
" 12,000 S10\n",
|
||||
" 000\n",
|
||||
" 0o0 SO\n",
|
||||
" 3,000 10\n",
|
||||
" Resource (BBOE)\n",
|
||||
" Delaware Basin Midland Basin Eagle Ford Bakken Other\n",
|
||||
" Largest Lower 48 Unconventional Producer; Growing into the Next Decade\n",
|
||||
" onshore operated inventory that achieves 15% IRR at $SO/BBL WTI, Competitors include CVX, DVN, EOG, FANG, MRO, OXY, PXD,and XOM:\n",
|
||||
" Source: Wood Mackenzie Lower 48 Unconventional Plays 2022 ProductionCompetitors include CVX, DVN; EOG, FANG, MRO, OXY, PXD and XOM; greaterthan50% liquids weight: ?Source: Wood Mackenzie (March 2023), Lower 48\n",
|
||||
" ConocoPhillips\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(base_response.source_nodes[0].get_content(metadata_mode=\"llm\"))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "llama_index_v3",
|
||||
"language": "python",
|
||||
"name": "llama_index_v3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 271 KiB |
+17
-2
@@ -126,6 +126,18 @@ class LlamaParse(BasePydanticReader):
|
||||
default=True,
|
||||
description="Whether to split by page (NOTE: using a predefined separator `\n---\n`)",
|
||||
)
|
||||
vendor_multimodal_api_key: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The API key for the multimodal API.",
|
||||
)
|
||||
use_vendor_multimodal_model: bool = Field(
|
||||
default=False,
|
||||
description="Whether to use the vendor multimodal API.",
|
||||
)
|
||||
vendor_multimodal_model_name: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The model name for the vendor multimodal API.",
|
||||
)
|
||||
|
||||
@validator("api_key", pre=True, always=True)
|
||||
def validate_api_key(cls, v: str) -> str:
|
||||
@@ -163,9 +175,9 @@ class LlamaParse(BasePydanticReader):
|
||||
file_name = extra_info["file_name"]
|
||||
mime_type = mimetypes.guess_type(file_name)[0]
|
||||
files = {"file": (file_name, file_input, mime_type)}
|
||||
elif isinstance(file_input, str):
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
file_path = str(file_input)
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if file_ext not in SUPPORTED_FILE_TYPES:
|
||||
raise Exception(
|
||||
f"Currently, only the following file types are supported: {SUPPORTED_FILE_TYPES}\n"
|
||||
@@ -199,6 +211,9 @@ class LlamaParse(BasePydanticReader):
|
||||
"gpt4o_api_key": self.gpt4o_api_key,
|
||||
"bounding_box": self.bounding_box,
|
||||
"target_pages": self.target_pages,
|
||||
"vendor_multimodal_api_key": self.vendor_multimodal_api_key,
|
||||
"use_vendor_multimodal_model": self.use_vendor_multimodal_model,
|
||||
"vendor_multimodal_model_name": self.vendor_multimodal_model_name,
|
||||
},
|
||||
)
|
||||
if not response.is_success:
|
||||
|
||||
Generated
+695
-693
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "llama-parse"
|
||||
version = "0.4.5"
|
||||
version = "0.4.8"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = ["Logan Markewich <logan@llamaindex.ai>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in New Issue
Block a user