mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-22 04:25:22 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Logan Markewich
|
7455c88d4a |
@@ -4,8 +4,9 @@ LlamaParse is an API created by LlamaIndex to efficiently parse and represent fi
|
||||
|
||||
LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).
|
||||
|
||||
Currently available in preview mode for **free**. Try it out today!
|
||||
|
||||
Free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page.
|
||||
**NOTE:** Currently, only PDF files are supported.
|
||||
|
||||
## Getting Started
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,615 +0,0 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# LlamaParse - Parsing comic books with parsing intructions\n",
|
||||
"Parsing intructions allow you to instruct our parsing model the same way you would instruct an LLM!\n",
|
||||
"\n",
|
||||
"They can be usefull to help the parser get better results on complex document layouts, to extract data in a specific format, or to transform the document in other ways.\n",
|
||||
"\n",
|
||||
"Using Parsing Instruction you will get better results out of LlamaParse on complicated documents, and also be able to simplify your application code."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "eld1dKaN7P8B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Parsing instructions are part of the llamaParse API. They can be accessed by directly specifying the parsing_instruction parameter in the API or by using the LlamaParse python module (which we will use for this tutorial).\n",
|
||||
"\n",
|
||||
"To install llama-parse, just get it from PIP:"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "goB1sV8zu_Xl"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"!pip install llama-parse"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "7Y3_BwQLu-qK",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"outputId": "652f8957-ae7a-48e3-86ae-2e9e885c4e1e"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Collecting llama-parse\n",
|
||||
" Downloading llama_parse-0.3.8-py3-none-any.whl (6.7 kB)\n",
|
||||
"Collecting llama-index-core>=0.10.7 (from llama-parse)\n",
|
||||
" Downloading llama_index_core-0.10.19-py3-none-any.whl (15.3 MB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m31.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (6.0.1)\n",
|
||||
"Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2.0.28)\n",
|
||||
"Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.9.3)\n",
|
||||
"Collecting dataclasses-json (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n",
|
||||
"Collecting deprecated>=1.2.9.3 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
|
||||
"Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
|
||||
"Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2023.6.0)\n",
|
||||
"Collecting httpx (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hCollecting llamaindex-py-client<0.2.0,>=0.1.13 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading llamaindex_py_client-0.1.13-py3-none-any.whl (107 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.0/108.0 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.6.0)\n",
|
||||
"Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.2.1)\n",
|
||||
"Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.8.1)\n",
|
||||
"Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.25.2)\n",
|
||||
"Collecting openai>=1.1.0 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading openai-1.13.3-py3-none-any.whl (227 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.4/227.4 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.5.3)\n",
|
||||
"Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (9.4.0)\n",
|
||||
"Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2.31.0)\n",
|
||||
"Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (8.2.3)\n",
|
||||
"Collecting tiktoken>=0.3.3 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (4.66.2)\n",
|
||||
"Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (4.10.0)\n",
|
||||
"Collecting typing-inspect>=0.8.0 (from llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
|
||||
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.3.1)\n",
|
||||
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (23.2.0)\n",
|
||||
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.4.1)\n",
|
||||
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (6.0.5)\n",
|
||||
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.9.4)\n",
|
||||
"Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (4.0.3)\n",
|
||||
"Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.9.3->llama-index-core>=0.10.7->llama-parse) (1.14.1)\n",
|
||||
"Requirement already satisfied: pydantic>=1.10 in /usr/local/lib/python3.10/dist-packages (from llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (2.6.3)\n",
|
||||
"Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (3.7.1)\n",
|
||||
"Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (2024.2.2)\n",
|
||||
"Collecting httpcore==1.* (from httpx->llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (3.6)\n",
|
||||
"Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (1.3.1)\n",
|
||||
"Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx->llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (8.1.7)\n",
|
||||
"Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (1.3.2)\n",
|
||||
"Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (2023.12.25)\n",
|
||||
"Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core>=0.10.7->llama-parse) (1.7.0)\n",
|
||||
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core>=0.10.7->llama-parse) (3.3.2)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core>=0.10.7->llama-parse) (2.0.7)\n",
|
||||
"Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core>=0.10.7->llama-parse) (3.0.3)\n",
|
||||
"Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
|
||||
"Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core>=0.10.7->llama-parse)\n",
|
||||
" Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n",
|
||||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||||
"\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core>=0.10.7->llama-parse) (2.8.2)\n",
|
||||
"Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core>=0.10.7->llama-parse) (2023.4)\n",
|
||||
"Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core>=0.10.7->llama-parse) (1.2.0)\n",
|
||||
"Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core>=0.10.7->llama-parse) (23.2)\n",
|
||||
"Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (0.6.0)\n",
|
||||
"Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (2.16.3)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->llama-index-core>=0.10.7->llama-parse) (1.16.0)\n",
|
||||
"Installing collected packages: dirtyjson, mypy-extensions, marshmallow, h11, deprecated, typing-inspect, tiktoken, httpcore, httpx, dataclasses-json, openai, llamaindex-py-client, llama-index-core, llama-parse\n",
|
||||
"Successfully installed dataclasses-json-0.6.4 deprecated-1.2.14 dirtyjson-1.0.8 h11-0.14.0 httpcore-1.0.4 httpx-0.27.0 llama-index-core-0.10.19 llama-parse-0.3.8 llamaindex-py-client-0.1.13 marshmallow-3.21.1 mypy-extensions-1.0.0 openai-1.13.3 tiktoken-0.6.0 typing-inspect-0.9.0\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## API key\n",
|
||||
"\n",
|
||||
"The use of LlamaParse requires an API key which you can get here: https://cloud.llamaindex.ai/parse"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "i-Rg2D_Rvf2i"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "af6i2P1vuU-U"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Async (Notebook only)\n",
|
||||
"llama-parse is async-first, so running the code in a notebook requires the use of nest_asyncio\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "p8Eq-aX-wAEo"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"import nest_asyncio\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "4OB0BkTqv_0l"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Import the package"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "dz927ecMyYo_"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"from llama_parse import LlamaParse"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "nSW-6sEwyXwx"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Using llamaparse for getting better results (on Manga!)\n",
|
||||
"\n",
|
||||
"Sometimes the layout of a page is unusual and you will get sub-optimal reading order results with LlamaParse. For example, when parsing manga you expect the reading order to be right to left even if the content is in English!"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "l_D4YsAHwUSk"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Let's download an extract of a great manga \"The manga guide to calculus\", by Hiroyuki Kojima (https://www.amazon.com/Manga-Guide-Calculus-Hiroyuki-Kojima/dp/1593271948)\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "SV4K2RivxzJG"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"! wget \"https://drive.usercontent.google.com/uc?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\" -O ./manga.pdf"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "d3qeuiyawT0U",
|
||||
"outputId": "e6da0635-dea2-4f2b-ec03-d8db99a75e17"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"--2024-03-13 13:57:19-- https://drive.usercontent.google.com/uc?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\n",
|
||||
"Resolving drive.usercontent.google.com (drive.usercontent.google.com)... 173.194.211.132, 2607:f8b0:400c:c10::84\n",
|
||||
"Connecting to drive.usercontent.google.com (drive.usercontent.google.com)|173.194.211.132|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 303 See Other\n",
|
||||
"Location: https://drive.usercontent.google.com/download?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download [following]\n",
|
||||
"--2024-03-13 13:57:19-- https://drive.usercontent.google.com/download?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\n",
|
||||
"Reusing existing connection to drive.usercontent.google.com:443.\n",
|
||||
"HTTP request sent, awaiting response... 200 OK\n",
|
||||
"Length: 3041634 (2.9M) [application/octet-stream]\n",
|
||||
"Saving to: ‘./manga.pdf’\n",
|
||||
"\n",
|
||||
"./manga.pdf 100%[===================>] 2.90M --.-KB/s in 0.04s \n",
|
||||
"\n",
|
||||
"2024-03-13 13:57:20 (78.6 MB/s) - ‘./manga.pdf’ saved [3041634/3041634]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Without parsing instructions\n",
|
||||
"For the sake of comparison, let's first parse without any instructions."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "Gbr8RiHEyF3-"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"vanilaParsing = LlamaParse(result_type=\"markdown\").load_data(\"./manga.pdf\")"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "3jKnXCuAyQ9_",
|
||||
"outputId": "8ab58d56-8c51-44de-8d34-0d94b1bb440f"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 25bf4202-78d8-4705-88cf-c616ae7c82af\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"As you can see below, LlamaParse is not doing a great job here. It is interpreting the grid of comic panels as a table, and trying to fit the dialogue into a table. It's very hard to follow."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "p4GVOdWzzvYg"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"print(vanilaParsing[0].text[100:1000])"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "ZMhWfKrzzhgQ",
|
||||
"outputId": "5426c0cc-7e62-4836-9877-87258e1f0b6f"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"\n",
|
||||
"The Asagake Times Sanda-Cho Distributor\n",
|
||||
"\n",
|
||||
"A newspaper distributor? do I have the wrong map?\n",
|
||||
"\n",
|
||||
"You’re looking It’s next for the Sanda-cho door. branch office? Everybody mistakes us for the office because we are larger. What Is a Function? 3\n",
|
||||
"---\n",
|
||||
"## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
|
||||
"\n",
|
||||
"|1.|Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is|\n",
|
||||
"|---|---|\n",
|
||||
"| |lim ε→0 (f(a + ε) - f(a)) / ε = lim ε→0 (α - α) = lim ε→0 0 = 0|\n",
|
||||
"| |Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.|\n",
|
||||
"\n",
|
||||
"Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
|
||||
"\n",
|
||||
"|2.|Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is|\n",
|
||||
"|---|---|\n",
|
||||
"| |lim ε→0 (f(α + ε) - f(a)) = \n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Using parsing instructions\n",
|
||||
"Let's try to parse the manga with custom instructions:\n",
|
||||
"\n",
|
||||
"\"The provided document is a manga comic book. Most pages do NOT have title. It does not contain tables. Try to reconstruct the dialogue happening in a cohesive way.\"\n",
|
||||
"\n",
|
||||
"To do so just pass the parsing instruction as a parameter to LlamaParse:"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "sUq6znUryiu0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"parsingInstructionManga = \"\"\"The provided document is a manga comic book, most page do NOT have title.\n",
|
||||
"It does not contain table.\n",
|
||||
"Try to reconstruct the dialog happening in a cohesive way.\"\"\"\n",
|
||||
"withInstructionParsing = LlamaParse(result_type=\"markdown\", parsing_instruction=parsingInstructionManga).load_data(\"./manga.pdf\")"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "dEX7Mv9V0UvM",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"outputId": "184b77b7-7e3a-4991-f2c9-eb9105a35a7b"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 88ab273e-b2a7-4f84-8e72-e9367cf6b114\n",
|
||||
"."
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Let's see how it compare with page 3! We encourage you to play with the target page and explore other pages. As you will see, the parsing instruction allowed LlamaParse to make sense of the document!\n",
|
||||
"\n",
|
||||
"<img src=\"https://drive.usercontent.google.com/download?id=1M87rXTIZE8d5v7aHmVZVW6gW3eDGq6ks&authuser=0\" />\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "-UQcA-YW2kjd"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"target_page=1\n",
|
||||
"print(vanilaParsing[0].text.split('\\n---\\n')[target_page])\n",
|
||||
"print(\"\\n\\n------------------------------------------------------------\\n\\n\")\n",
|
||||
"print(withInstructionParsing[0].text.split('\\n---\\n')[target_page])"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "0oPHXg0F0yAS",
|
||||
"outputId": "38b729bc-b0b7-42f2-97e0-b4e9d7d459a1"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"The Asagake Times Sanda-Cho Distributor\n",
|
||||
"\n",
|
||||
"A newspaper distributor? do I have the wrong map?\n",
|
||||
"\n",
|
||||
"You’re looking It’s next for the Sanda-cho door. branch office? Everybody mistakes us for the office because we are larger. What Is a Function? 3\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# The Asagake Times\n",
|
||||
"\n",
|
||||
"Sanda-Cho Distributor\n",
|
||||
"\n",
|
||||
"A newspaper distributor?\n",
|
||||
"\n",
|
||||
"Do I have the wrong map?\n",
|
||||
"\n",
|
||||
"You're looking for the Sanda-cho branch office?\n",
|
||||
"\n",
|
||||
"It's next door.\n",
|
||||
"\n",
|
||||
"Everybody mistakes us for the office because we are larger.\n",
|
||||
"\n",
|
||||
"What Is a Function? 3\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Math - doing more with parsing instuction!\n",
|
||||
"\n",
|
||||
"But this manga is about math and full of equations, why not ask the parser to output them in **LaTeX**?\n",
|
||||
"\n",
|
||||
"<img src=\"https://drive.usercontent.google.com/download?id=1tze3xcQ7axVA-vC_iZeAj_GvYcyNuYDa&authuser=0\" />"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "yU_jyYWI5fMH"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"parsingInstructionMangaLatex = \"\"\"The provided document is a manga comic book, most page do NOT have title.\n",
|
||||
"It does not contain table. Do not output table.\n",
|
||||
"Try to reconstruct the dialog happening in a cohesive way.\n",
|
||||
"Output any math equation in LATEX markdown (between $$)\"\"\"\n",
|
||||
"withLatex = LlamaParse(result_type=\"markdown\", parsing_instruction=parsingInstructionMangaLatex).load_data(\"./manga.pdf\")"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "FP_YdO2y5e5o",
|
||||
"outputId": "45171cdd-298a-4f1f-e67d-6be3cb1a1156"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 3a055e64-d91e-484e-b9b0-99a2e637c08d\n",
|
||||
"."
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"target_page=2\n",
|
||||
"print(\"\\n\\n[Without instruction]------------------------------------------------------------\\n\\n\")\n",
|
||||
"print(vanilaParsing[0].text.split('\\n---\\n')[target_page])\n",
|
||||
"print(\"\\n\\n[With instruction to output math in LATEX!]------------------------------------------------------------\\n\\n\")\n",
|
||||
"print(withLatex[0].text.split('\\n---\\n')[target_page])\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "TntdRRGp6Rui",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"outputId": "e2e503fa-eb87-4f78-83d8-209fe7cebd9e"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"[Without instruction]------------------------------------------------------------\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
|
||||
"\n",
|
||||
"|1.|Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is|\n",
|
||||
"|---|---|\n",
|
||||
"| |lim ε→0 (f(a + ε) - f(a)) / ε = lim ε→0 (α - α) = lim ε→0 0 = 0|\n",
|
||||
"| |Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.|\n",
|
||||
"\n",
|
||||
"Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
|
||||
"\n",
|
||||
"|2.|Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is|\n",
|
||||
"|---|---|\n",
|
||||
"| |lim ε→0 (f(α + ε) - f(a)) = lim ε→0 (α(a + ε) + β - (αa + β)) = lim ε→0 α = α|\n",
|
||||
"| |Thus, the derivative of f(x) is f′(x) = α, a constant value. This result should also be intuitive—linear functions have a constant rate of change by definition.|\n",
|
||||
"\n",
|
||||
"|3.|Let’s find the derivative of f(x) = x^2, which appeared in the story. The differential coefficient of f(x) at x = a is|\n",
|
||||
"|---|---|\n",
|
||||
"| |lim ε→0 ((a + ε)^2 - a^2) / ε = lim (a^2 + 2aε + ε^2 - a^2) / ε = lim (2aε + ε^2) = lim (2a + ε) = 2a|\n",
|
||||
"| |Thus, the differential coefficient of f(x) at x = a is 2a, or f′(a) = 2a. Therefore, the derivative of f(x) is f′(x) = 2x.|\n",
|
||||
"\n",
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"- The calculation of a limit that appears in calculus is simply a formula calculating an error.\n",
|
||||
"- A limit is used to obtain a derivative.\n",
|
||||
"- The derivative is the slope of the tangent line at a given point.\n",
|
||||
"- The derivative is nothing but the rate of change.\n",
|
||||
"\n",
|
||||
"## Chapter 1 Let’s Differentiate a Function!\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"[With instruction to output math in LATEX!]------------------------------------------------------------\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Derivative of Constant, Linear, or Quadratic Function\n",
|
||||
"\n",
|
||||
"## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
|
||||
"\n",
|
||||
"1. Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(a + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\frac{\\alpha - \\alpha}{\\varepsilon} = \\lim_{{\\varepsilon \\to 0}} 0 = 0 \\\\\n",
|
||||
"\\end{align*}\n",
|
||||
"$$\n",
|
||||
"Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.\n",
|
||||
"\n",
|
||||
"Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
|
||||
"\n",
|
||||
"2. Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(\\alpha + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\frac{\\alpha(a + \\varepsilon) + \\beta - (\\alpha a + \\beta)}{\\varepsilon} = \\lim_{{\\varepsilon \\to 0}} \\alpha = \\alpha \\\\\n",
|
||||
"\\end{align*}\n",
|
||||
"$$\n",
|
||||
"Thus, the derivative of f(x) is f′(x) = α, a constant value. This result should also be intuitive—linear functions have a constant rate of change by definition.\n",
|
||||
"\n",
|
||||
"3. Let’s find the derivative of f(x) = x2. The differential coefficient of f(x) at x = a is\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(a + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\left( (a + \\varepsilon)^2 - a^2 \\right) = \\lim_{{\\varepsilon \\to 0}} 2a\\varepsilon + \\varepsilon = \\lim_{{\\varepsilon \\to 0}} (2a + \\varepsilon) = 2a \\\\\n",
|
||||
"\\end{align*}\n",
|
||||
"$$\n",
|
||||
"Thus, the differential coefficient of f(x) at x = a is 2a, or f′(a) = 2a. Therefore, the derivative of f(x) is f′(x) = 2x.\n",
|
||||
"\n",
|
||||
"### Summary\n",
|
||||
"\n",
|
||||
"- The calculation of a limit that appears in calculus is simply a formula calculating an error.\n",
|
||||
"- A limit is used to obtain a derivative.\n",
|
||||
"- The derivative is the slope of the tangent line at a given point.\n",
|
||||
"- The derivative is nothing but the rate of change.\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"And here is the result as rendered by https://upmath.me/ .\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"<img src=\"https://drive.usercontent.google.com/download?id=1qGo5bMGYOiIC9MnprcgEByaYjU9YII2Q&authuser=0\" />\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Over this short notebook we saw how to use parsing instructions to increase the quality and accuracy of parsing with LLamaParse!"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "rfFdeWZKmmLW"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,405 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e0647976-5597-4899-8678-e9a73c19f18b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# LlamaParse over Powerpoint Files\n",
|
||||
"\n",
|
||||
"In this notebook we show you how to build a RAG pipeline over [our talk at PyData Global](https://docs.google.com/presentation/d/1rFQ0hPyYja3HKRdGEgjeDxr0MSE8wiQ2iu4mDtwR6fc/edit?usp=sharing) in 2023.\n",
|
||||
"\n",
|
||||
"We use LlamaParse to load in our slides in .pptx format, and use LlamaIndex to build a RAG pipeline over these files.\n",
|
||||
"\n",
|
||||
"**NOTE**: LlamaParse is capable of image extraction through JSON mode, in this notebook we stick with text."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "14cdcfaf-88b4-4489-9910-e362e0ccec53",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import nest_asyncio\n",
|
||||
"nest_asyncio.apply()\n",
|
||||
"\n",
|
||||
"from llama_parse import LlamaParse"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "6f5b5841-dd3e-4169-9bd4-6a672b5b34ee",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a2a619a1-fdd4-4ff0-85f8-94c125c275eb",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Download Data\n",
|
||||
"\n",
|
||||
"First, download the slides from https://docs.google.com/presentation/d/1rFQ0hPyYja3HKRdGEgjeDxr0MSE8wiQ2iu4mDtwR6fc/edit?usp=sharing and export in .pptx format, and put it in the folder that you're running this notebook.\n",
|
||||
"\n",
|
||||
"Name the file `pydata_global.pptx`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a7e697d9-4463-4be4-908c-0a3e9179a342",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## [Basic] Build a RAG Pipeline over Powerpoint Text\n",
|
||||
"\n",
|
||||
"In this example, we use LlamaParse in markdown mode to extract out text from the slides, and we build a top-k RAG pipeline over it.\n",
|
||||
"\n",
|
||||
"**Notes**: \n",
|
||||
"- This does not use our `MarkdownElementNodeParser`, which is tailored for documents with tables.\n",
|
||||
"- This also does not parse out images (we show that in the next section).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "0dd0f860-8e92-43a7-9443-ad1a4fb9365c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"parser = LlamaParse(result_type=\"markdown\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "fd932bef-ba82-4449-b7a0-5c2a9b55089f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 9c687e37-4239-4c2f-b2a1-2564bfc98473\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"docs = parser.load_data(\"pydata_global.pptx\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0f41c2bc-02cd-49b5-a98c-f986faa8fffc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's take a look at a few slides."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "2a73e553-2194-4ac9-9764-0edab0d6fdce",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"# Building and Productionizing RAG\n",
|
||||
"\n",
|
||||
"Jerry Liu, LlamaIndex co-founder/CEO\n",
|
||||
"---\n",
|
||||
"|Content|Page Number|\n",
|
||||
"|---|---|\n",
|
||||
"|Document Processing| |\n",
|
||||
"|Tagging & Extraction| |\n",
|
||||
"|Knowledge Base| |\n",
|
||||
"|Knowledge Search & QA| |\n",
|
||||
"|Workflow:| |\n",
|
||||
"|Read latest messages from user A| |\n",
|
||||
"|Send email suggesting next-steps| |\n",
|
||||
"|Document| |\n",
|
||||
"|Human:| |\n",
|
||||
"|Agent:| |\n",
|
||||
"|Topic:| |\n",
|
||||
"|Summary:| |\n",
|
||||
"|Author:| |\n",
|
||||
"|Conversational Agent| |\n",
|
||||
"|Workflow Automation| |\n",
|
||||
"---\n",
|
||||
"Context\n",
|
||||
"\n",
|
||||
"- LLMs are a phenomenal piece of technology for knowledge generation and reasoning. They are pre-trained on large amounts of publicly available data.\n",
|
||||
"\n",
|
||||
"Use Cases\n",
|
||||
"\n",
|
||||
"- Question-Answering\n",
|
||||
"- Text Generation\n",
|
||||
"- Summarization\n",
|
||||
"- Planning\n",
|
||||
"\n",
|
||||
"# LLM’s\n",
|
||||
"---\n",
|
||||
"|Context|\n",
|
||||
"|---|\n",
|
||||
"|How do we best augment LLMs with our own private data?|\n",
|
||||
"|Raw Files|API’s|\n",
|
||||
"| |salesforce|?|\n",
|
||||
"| | |Use Cases|\n",
|
||||
"| | |Question-Answering|\n",
|
||||
"| | |Text Generation|\n",
|
||||
"| | |Summarization|\n",
|
||||
"|Vector Stores|SQL DB’s|\n",
|
||||
"| | |Planning|\n",
|
||||
"| |LLM’s|\n",
|
||||
"| |Milvus|\n",
|
||||
"---\n",
|
||||
"Paradigms for inserting knowledge\n",
|
||||
"\n",
|
||||
"Retrieval Augmentation - Fix pe model, put context into pe prompt\n",
|
||||
"Before college pe two main pings I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write pen, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters wip strong feelings, which I imagined made pem deep...\n",
|
||||
"\n",
|
||||
"Input Prompt\n",
|
||||
"\n",
|
||||
"Here is the context:\n",
|
||||
"\n",
|
||||
"Before college the two main things...\n",
|
||||
"\n",
|
||||
"Given the context, answer the following question:\n",
|
||||
"\n",
|
||||
"{query_str} LLM\n",
|
||||
"---\n",
|
||||
"Paradigms for inserting knowledge\n",
|
||||
"\n",
|
||||
"Fine-tuning - baking knowledge into pe weights of pe network\n",
|
||||
"Before college pe two main pings I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write pen, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters wip strong feelings, which I imagined made pem deep... LLM RLHF, Adam, SGD, etc.\n",
|
||||
"---\n",
|
||||
"## LlamaIndex: A data framework for LLM applications\n",
|
||||
"\n",
|
||||
"|Data Ingestion (LlamaHub 🦙)|Data Structures|Queries|\n",
|
||||
"|---|---|---|\n",
|
||||
"|Connect your existing data sources and data formats (API’s, PDF’s, docs, SQL, etc.)|Store and index your data for different use cases. Integrate with different db’s (vector db, graph db, kv db)|Retrieve and query over data. Includes: QA, Summarization, Agents, and more|\n",
|
||||
"---\n",
|
||||
"# quickstart py\n",
|
||||
"\n",
|
||||
"from Llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
|
||||
"\n",
|
||||
"SimpleDirectoryReader( ' data' ) . Load_datal)\n",
|
||||
"\n",
|
||||
"documents\n",
|
||||
"\n",
|
||||
"VectorStoreIndex.from_documents\n",
|
||||
"\n",
|
||||
"indexdocuments_\n",
|
||||
"\n",
|
||||
"index.as_query_engine()\n",
|
||||
"\n",
|
||||
"query_engine\n",
|
||||
"\n",
|
||||
"query_engine.query ( \"What did the authordo growingup?\" )\n",
|
||||
"\n",
|
||||
"response\n",
|
||||
"\n",
|
||||
"print(str(response) )Codelmage\n",
|
||||
"---\n",
|
||||
"NO_CONTENT_HERE\n",
|
||||
"---\n",
|
||||
"|Data Ingestion / Parsing|Data Querying|\n",
|
||||
"|---|---|\n",
|
||||
"|Chunk| |\n",
|
||||
"|Chunk| |\n",
|
||||
"|Doc|Chunk|\n",
|
||||
"|Chunk|Chunk|\n",
|
||||
"| |Vector|Chunk|LLM|\n",
|
||||
"| | |Database|\n",
|
||||
"|Chunk| |\n",
|
||||
"| |5 Lines of Code in LlamaIndex!|\n",
|
||||
"---\n",
|
||||
"|Current RAG Stack (Data Ingestion/Parsing)|Process:|\n",
|
||||
"|---|---|\n",
|
||||
"|● Split up document(s) into even chunks.| |\n",
|
||||
"|● Each chunk is a piece of raw text.| |\n",
|
||||
"|Chunk|● Generate embedding for each chunk (e.g. OpenAI embeddings, sentence_transformer)|\n",
|
||||
"|Chunk|● Store each chunk into a vector database|\n",
|
||||
"|Doc|Chunk|\n",
|
||||
"|Chunk|Vector Database|\n",
|
||||
"|Chunk| |\n",
|
||||
"---\n",
|
||||
"|Current RAG Stack (Querying)|\n",
|
||||
"|---|\n",
|
||||
"|Process:|\n",
|
||||
"|● Find top-k most similar chunks from vector database collection|\n",
|
||||
"|● Plug into LLM response synthesis module|\n",
|
||||
"|Chunk|Chunk|LLM|\n",
|
||||
"|Vector|Chunk| |\n",
|
||||
"|Database|\n",
|
||||
"---\n",
|
||||
"|Current RAG Stack (Querying)|\n",
|
||||
"|---|\n",
|
||||
"|Process:|\n",
|
||||
"|● Find top-k most similar chunks from vector database collection|\n",
|
||||
"|● Plug into LLM response synthesis module|\n",
|
||||
"|Chunk|LLM|\n",
|
||||
"|Chunk|\n",
|
||||
"|Vector|\n",
|
||||
"|Database|\n",
|
||||
"|Retrieval|Synthesis|\n",
|
||||
"---\n",
|
||||
"|Query|Nodel|Response|Nodez|\n",
|
||||
"|---|---|---|---|\n",
|
||||
"|Create and Refine|Intermediate| | |\n",
|
||||
"| | |Final|Response|\n",
|
||||
"---\n",
|
||||
"|Query|Node1|Node2|Node3|Node4|\n",
|
||||
"|---|---|---|---|---|\n",
|
||||
"|Tree Summarize| | | | |\n",
|
||||
"---\n",
|
||||
"Quickstart\n",
|
||||
"\n",
|
||||
"Link to Google Colab\n",
|
||||
"---\n",
|
||||
"NO_CONTENT_HERE\n",
|
||||
"---\n",
|
||||
"# Challenges with Naive RAG\n",
|
||||
"\n",
|
||||
"- Failure Modes\n",
|
||||
"- Quality-Related (Hallucination, Accuracy)\n",
|
||||
"- Non-Quality-Related (Latency, Cost, Syncing)\n",
|
||||
"---\n",
|
||||
"## Challenges with Naive RAG (Response Quality)\n",
|
||||
"\n",
|
||||
"|Bad Retrieval|Low Precision: Not all chunks in retrieved set are relevant|Hallucination + Lost in the Middle Problems|\n",
|
||||
"|---|---|---|\n",
|
||||
"| |Low Recall: Now all relevant chunks are retrieved.|Lacks enough context for LLM to synthesize an answer|\n",
|
||||
"| |Outdated information: The data is redundant or out of date.| |\n",
|
||||
"---\n",
|
||||
"## Challenges with Naive RAG (Response Quality)\n",
|
||||
"\n",
|
||||
"|Bad Retrieval|Low Precision: Not all chunks in retrieved set are relevant|Hallucination + Lost in the Middle Problems|\n",
|
||||
"|---|---|---|\n",
|
||||
"| |Low Recall: Now all relevant chunks are retrieved.|Lacks enough context for LLM to synthesize an answer|\n",
|
||||
"| |Outdated information: The data is redundant or out of date.| |\n",
|
||||
"|Bad Response Generation|Hallucination: Model makes up an answer that isn’t in the context.| |\n",
|
||||
"| |Irrelevance: Model makes up an answer that doesn’t answer the question.| |\n",
|
||||
"| |Toxicity/Bias: Model makes up an answer t\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(docs[0].get_content()[:5000])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c2fa0a1a-1ed8-4a5a-a0c1-5792fe32634b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Build a RAG pipeline over these documents\n",
|
||||
"\n",
|
||||
"We now use LlamaIndex to build a RAG pipeline over these powerpoint slides."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "c779547f-e4f7-4c84-9786-2b6b749827ab",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import VectorStoreIndex"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "68b3a95e-ce19-4df1-9fdd-e6efb2fc423a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"index = VectorStoreIndex.from_documents(docs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "a2ae28f6-4b3a-4130-8e65-0921b7678739",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_engine = index.as_query_engine()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "232091ee-aa22-4f51-838c-410024acc344",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"response = query_engine.query(\"What are some response quality challenges with naive RAG?\") "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "75f32aa7-c308-4221-af60-779822cfdba1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some response quality challenges with naive RAG include issues related to bad retrieval, such as low precision where not all retrieved chunks are relevant, leading to problems like hallucination and being lost in the middle. Additionally, low recall can occur when not all relevant chunks are retrieved, resulting in a lack of sufficient context for the language model to synthesize an answer. Outdated information in the retrieved data can also pose a challenge. On the response generation side, challenges include hallucination where the model generates an answer not present in the context, irrelevance where the answer does not address the question, and toxicity/bias where the answer is harmful or offensive.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(str(response))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d309d8fb-750a-4393-a1b2-67b14b7c121f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "llama_parse",
|
||||
"language": "python",
|
||||
"name": "llama_parse"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1,422 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "eld1dKaN7P8B"
|
||||
},
|
||||
"source": [
|
||||
"# LlamaParse - Parsing Financial Powerpoints 📊\n",
|
||||
"\n",
|
||||
"In this cookbook we show you how to use LlamaParse to parse a financial powerpoint."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "goB1sV8zu_Xl"
|
||||
},
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Parsing instruction are part of the LlamaParse API. They can be access by directly specifying the parsing_instruction parameter in the API or by using LlamaParse python module (which we will use for this tutorial).\n",
|
||||
"\n",
|
||||
"To install llama-parse, just get it from `pip`:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "7Y3_BwQLu-qK",
|
||||
"outputId": "b1129c52-7a70-44cc-ad03-1f8d3a8c794a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install llama-index\n",
|
||||
"!pip install llama-parse\n",
|
||||
"!pip install torch transformers python-pptx Pillow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "i-Rg2D_Rvf2i"
|
||||
},
|
||||
"source": [
|
||||
"## API Key\n",
|
||||
"\n",
|
||||
"The use of LlamaParse requires an API key which you can get here: https://cloud.llamaindex.ai/parse"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "af6i2P1vuU-U"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
|
||||
"os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "p8Eq-aX-wAEo"
|
||||
},
|
||||
"source": [
|
||||
"**NOTE**: Since LlamaParse is natively async, running the sync code in a notebook requires the use of nest_asyncio.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"id": "4OB0BkTqv_0l",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import nest_asyncio\n",
|
||||
"\n",
|
||||
"nest_asyncio.apply()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "dz927ecMyYo_"
|
||||
},
|
||||
"source": [
|
||||
"## Importing the package\n",
|
||||
"\n",
|
||||
"To import llama_parse simply do:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "nSW-6sEwyXwx",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_parse import LlamaParse"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "l_D4YsAHwUSk"
|
||||
},
|
||||
"source": [
|
||||
"## Using LlamaParse to Parse Presentations\n",
|
||||
"\n",
|
||||
"Like Powerpoints, presentations are often hard to extract for RAG. With LlamaParse we can now parse them and unclock their content of presentations for RAG.\n",
|
||||
"\n",
|
||||
"Let's download a financial report from the World Meteorological Association."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "d3qeuiyawT0U",
|
||||
"outputId": "cec0ea0a-be8b-49b6-9376-797c91f63be7",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! mkdir data; wget \"https://meetings.wmo.int/Cg-19/PublishingImages/SitePages/FINAC-43/7%20-%20EC-77-Doc%205%20Financial%20Statements%20for%202022%20(FINAC).pptx\" -O data/presentation.pptx"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Gbr8RiHEyF3-"
|
||||
},
|
||||
"source": [
|
||||
"### Parsing the presentation\n",
|
||||
"\n",
|
||||
"Now let's parse it into Markdown with LlamaParse and the default LlamaIndex parser.\n",
|
||||
"\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "osocsofoJ42S"
|
||||
},
|
||||
"source": [
|
||||
"#### Llama Index default"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "PTVy5XCNJwW-",
|
||||
"outputId": "d0e2cc4b-1407-45a9-b5e6-d06f91a533b4",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import SimpleDirectoryReader\n",
|
||||
"\n",
|
||||
"vanilla_documents = SimpleDirectoryReader(\"./data/\").load_data()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "oucbsciZJwxt"
|
||||
},
|
||||
"source": [
|
||||
"#### Llama Parse"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "3jKnXCuAyQ9_",
|
||||
"outputId": "1f668f17-1e20-46e5-fbab-9a55e4b28891",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Started parsing the file under job_id 56724c0d-e45a-4e30-ae8c-e416173c608a\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"llama_parse_documents = LlamaParse(result_type=\"markdown\").load_data(\"./data/presentation.pptx\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's take a look at the parsed output from an example slide (see image below).\n",
|
||||
"\n",
|
||||
"As we can see the table is faithfully extracted!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"ation and mitigation\n",
|
||||
"---\n",
|
||||
"|Item|31 Dec 2022|31 Dec 2021|Change|\n",
|
||||
"|---|---|---|---|\n",
|
||||
"|Payables and accruals|4,685|4,066|619|\n",
|
||||
"|Employee benefits|127,215|84,676|42,539|\n",
|
||||
"|Contributions received in advance|6,975|10,192|(3,217)|\n",
|
||||
"|Unearned revenue from exchange transactions|20|651|(631)|\n",
|
||||
"|Deferred Revenue|71,301|55,737|15,564|\n",
|
||||
"|Borrowings|28,229|29,002|(773)|\n",
|
||||
"|Funds held in trust|30,373|29,014|1,359|\n",
|
||||
"|Provisions|1,706|1,910|(204)|\n",
|
||||
"|Total Liabilities|270,504|215,248|55,256|\n",
|
||||
"---\n",
|
||||
"## Liabilities\n",
|
||||
"\n",
|
||||
"Employee Ben\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(llama_parse_documents[0].get_content()[-2800:-2300])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"Compared against the original slide image.\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "p4GVOdWzzvYg"
|
||||
},
|
||||
"source": [
|
||||
"## Comparing the two for RAG\n",
|
||||
"\n",
|
||||
"The main difference between LlamaParse and the previous directory reader approach, it that LlamaParse will extract the document in a structured format, allowing better RAG."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "oVcdGus5NDxi"
|
||||
},
|
||||
"source": [
|
||||
"### Query Engine on SimpleDirectoryReader results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {
|
||||
"id": "DqXYsLCWNg9_",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
|
||||
"\n",
|
||||
"vanilla_index = VectorStoreIndex.from_documents(vanilla_documents)\n",
|
||||
"vanilla_query_engine = vanilla_index.as_query_engine()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ZLkHt9l2Nbxx"
|
||||
},
|
||||
"source": [
|
||||
"### Query Engine on LlamaParse Results\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"metadata": {
|
||||
"id": "ZllaDcfRNLv3",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"llama_parse_index = VectorStoreIndex.from_documents(llama_parse_documents)\n",
|
||||
"llama_parse_query_engine = llama_parse_index.as_query_engine()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "0dY_0_1bNg0X",
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"### Liability provision\n",
|
||||
"What was the liability provision as of Dec 31 2021?\n",
|
||||
"\n",
|
||||
"<!-- <img src=\"https://drive.usercontent.google.com/download?id=184jVq0QyspDnmCyRfV0ebmJJxmAOJHba&authuser=0\" /> -->"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "Tmn-qNTEN-cb",
|
||||
"outputId": "a9bffc00-9cfc-43d8-b159-596a6c1aca64",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The liability provision as of December 31, 2021, included Employee Benefit Liabilities, Contributions received in advance (assessed contributions), and Deferred revenue.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"vanilla_response = vanilla_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
|
||||
"print(vanilla_response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "4EZ_uqlROP7R",
|
||||
"outputId": "0645a159-06c6-411e-d1f6-79ea95d32b42",
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The liability provision as of December 31, 2021, was 1,910 CHF.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"llama_parse_response = llama_parse_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
|
||||
"print(llama_parse_response)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "llama_parse",
|
||||
"language": "python",
|
||||
"name": "llama_parse"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
+7
-24
@@ -108,19 +108,6 @@ class Language(str, Enum):
|
||||
VIETNAMESE = "vi"
|
||||
|
||||
|
||||
SUPPORTED_FILE_TYPES = [
|
||||
".pdf",
|
||||
".xml"
|
||||
".doc",
|
||||
".docx",
|
||||
".pptx",
|
||||
".rtf",
|
||||
".pages",
|
||||
".key",
|
||||
".epub"
|
||||
]
|
||||
|
||||
|
||||
class LlamaParse(BasePydanticReader):
|
||||
"""A smart-parser for files."""
|
||||
|
||||
@@ -178,12 +165,8 @@ class LlamaParse(BasePydanticReader):
|
||||
# upload a document and get back a job_id
|
||||
async def _create_job(self, file_path: str, extra_info: Optional[dict] = None) -> str:
|
||||
file_path = str(file_path)
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
if file_ext not in SUPPORTED_FILE_TYPES:
|
||||
raise Exception(
|
||||
f"Currently, only the following file types are supported: {SUPPORTED_FILE_TYPES}\n"
|
||||
f"Current file type: {file_ext}"
|
||||
)
|
||||
if not file_path.endswith(".pdf"):
|
||||
raise Exception("Currently, only PDF files are supported.")
|
||||
|
||||
extra_info = extra_info or {}
|
||||
extra_info["file_path"] = file_path
|
||||
@@ -200,7 +183,7 @@ class LlamaParse(BasePydanticReader):
|
||||
async with httpx.AsyncClient(timeout=self.max_timeout) as client:
|
||||
response = await client.post(url, files=files, headers=headers, data={"language": self.language.value, "parsing_instruction": self.parsing_instruction})
|
||||
if not response.is_success:
|
||||
raise Exception(f"Failed to parse the file: {response.text}")
|
||||
raise Exception(f"Failed to parse the PDF file: {response.text}")
|
||||
|
||||
# check the status of the job, return when done
|
||||
job_id = response.json()["id"]
|
||||
@@ -223,7 +206,7 @@ class LlamaParse(BasePydanticReader):
|
||||
end = time.time()
|
||||
if end - start > self.max_timeout:
|
||||
raise Exception(
|
||||
f"Timeout while parsing the file: {job_id}"
|
||||
f"Timeout while parsing the PDF file: {job_id}"
|
||||
)
|
||||
if self.verbose and tries % 10 == 0:
|
||||
print(".", end="", flush=True)
|
||||
@@ -231,7 +214,7 @@ class LlamaParse(BasePydanticReader):
|
||||
|
||||
if result.status_code == 400:
|
||||
detail = result.json().get("detail", "Unknown error")
|
||||
raise Exception(f"Failed to parse the file: {detail}")
|
||||
raise Exception(f"Failed to parse the PDF file: {detail}")
|
||||
|
||||
return result.json()
|
||||
|
||||
@@ -252,7 +235,7 @@ class LlamaParse(BasePydanticReader):
|
||||
]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error while parsing the file '{file_path}':", e)
|
||||
print(f"Error while parsing the PDF file '{file_path}':", e)
|
||||
raise e
|
||||
return []
|
||||
|
||||
@@ -300,7 +283,7 @@ class LlamaParse(BasePydanticReader):
|
||||
return [result]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error while parsing the file '{file_path}':", e)
|
||||
print(f"Error while parsing the PDF file '{file_path}':", e)
|
||||
raise e
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "llama-parse"
|
||||
version = "0.3.9"
|
||||
version = "0.3.8"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = ["Logan Markewich <logan@llamaindex.ai>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in New Issue
Block a user