cr

* ADD: Support for language parameter in the package

* add language enum

README.md

@@ -1,4 +1,4 @@
-# LlamaParse (Preview)
+# LlamaParse
 
 LlamaParse is an API created by LlamaIndex to efficiently parse and represent files for efficient retrieval and context augmentation using LlamaIndex frameworks.
 
@@ -12,11 +12,20 @@ Currently available in preview mode for **free**. Try it out today!
 
 First, login and get an api-key from `https://cloud.llamaindex.ai`.
 
-Install the package:
+Then, make sure you have the latest LlamaIndex version installed.
+
+**NOTE:** If you are upgrading from v0.9.X, we recommend following our [migration guide](https://pretty-sodium-5e0.notion.site/v0-10-0-Migration-Guide-6ede431dcb8841b09ea171e7f133bd77), as well as uninstalling your previous version first.
+
+```
+pip uninstall llama-index  # run this if upgrading from v0.9.x or older
+pip install -U llama-index --upgrade --no-cache-dir --force-reinstall
+```
+
+Lastly, install the package:
 
 `pip install llama-parse`
 
-Then, you can run the following to parse your first PDF file:
+Now you can run the following to parse your first PDF file:
 
 ```python
 import nest_asyncio
@@ -28,7 +37,8 @@ parser = LlamaParse(
     api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
     result_type="markdown",  # "markdown" and "text" are available
     num_workers=4, # if multiple files passed, split in `num_workers` API calls
-    verbose=True
+    verbose=True,
+    language="en" # Optionaly you can define a language, default=en
 )
 
 # sync

examples/demo_advanced.ipynb

@@ -18,7 +18,7 @@
    "outputs": [],
    "source": [
     "!pip install llama-index\n",
-    "!pip install llama-index-core==0.10.6.post1\n",
+    "!pip install llama-index-core\n",
     "!pip install llama-index-embeddings-openai\n",
     "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
     "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
@@ -489,7 +489,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "llama-parse-aNC435Vv-py3.11",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -503,10 +503,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
-  },
-  "orig_nbformat": 4
+   "version": "3.10.10"
+  }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

examples/demo_astradb.ipynb

@@ -0,0 +1,296 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Using llama-parse with AstraDB"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this notebook, we show a basic RAG-style example that uses `llama-parse` to parse a PDF document, store the corresponding document into a vector store (`AstraDB`) and finally, perform some basic queries against that store. The notebook is modeled after the quick start notebooks and hence is meant as a way of getting started with `llama-parse`, backed by a vector database."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Requirements"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# First, install the required dependencies\n",
+    "!pip install --quiet llama-index llama-parse llama-index-vector-stores-astra llama-index-llms-openai astrapy"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import openai\n",
+    "\n",
+    "from getpass import getpass\n",
+    "\n",
+    "# Get all required API keys and parameters\n",
+    "llama_cloud_api_key = getpass(\"Enter your Llama Index Cloud API Key: \")\n",
+    "api_endpoint = input(\"Enter your Astra DB API Endpoint: \")\n",
+    "token = getpass(\"Enter your Astra DB Token: \")\n",
+    "namespace = input(\"Enter your Astra DB namespace (optional, must exist on Astra): \") or None\n",
+    "openai_api_key = getpass(\"Enter your OpenAI API Key: \")\n",
+    "\n",
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = llama_cloud_api_key\n",
+    "openai.api_key = openai_api_key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Using llama-parse to parse a PDF"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Download complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Grab a PDF from Arxiv for indexing\n",
+    "import requests \n",
+    "\n",
+    "# The URL of the file you want to download\n",
+    "url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
+    "# The local path where you want to save the file\n",
+    "file_path = \"./attention.pdf\"\n",
+    "\n",
+    "# Perform the HTTP request\n",
+    "response = requests.get(url)\n",
+    "\n",
+    "# Check if the request was successful\n",
+    "if response.status_code == 200:\n",
+    "    # Open the file in binary write mode and save the content\n",
+    "    with open(file_path, \"wb\") as file:\n",
+    "        file.write(response.content)\n",
+    "    print(\"Download complete.\")\n",
+    "else:\n",
+    "    print(\"Error downloading the file.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id ce3909a7-54cf-438b-849a-fe9a903b0c71\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_parse import LlamaParse\n",
+    "\n",
+    "documents = LlamaParse(result_type=\"text\").load_data(file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'rmer - model architecture.\\nThe Transformer follows this overall architecture using stacked self-attention and point-wise, fully\\nconnected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\\nrespectively.\\n3.1   Encoder and Decoder Stacks\\nEncoder:     The encoder is composed of a stack of N = 6 identical layers. Each layer has two\\nsub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\\nwise fully connected feed-forward network. We employ a residual connection [11] around each of\\nthe two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\\nLayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\\nitself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\\nlayers, produce outputs of dimension dmodel = 512.\\nDecoder:    The decoder is also composed of a stack of N = 6 identical layers. In addition '"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Take a quick look at some of the parsed text from the document:\n",
+    "documents[0].get_content()[10000:11000]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Storing into Astra DB"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.vector_stores.astra import AstraDBVectorStore\n",
+    "\n",
+    "astra_db_store = AstraDBVectorStore(\n",
+    "    token=token,\n",
+    "    api_endpoint=api_endpoint,\n",
+    "    namespace=namespace,\n",
+    "    collection_name=\"astra_v_table_llamaparse\",\n",
+    "    embedding_dimension=1536\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from llama_index.core.node_parser import SimpleNodeParser\n",
+    "\n",
+    "node_parser = SimpleNodeParser()\n",
+    "\n",
+    "nodes = node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.core import VectorStoreIndex, StorageContext\n",
+    "\n",
+    "storage_context = StorageContext.from_defaults(vector_store=astra_db_store)\n",
+    "\n",
+    "index = VectorStoreIndex(\n",
+    "    nodes=nodes,\n",
+    "    storage_context=storage_context,\n",
+    "    embed_model=OpenAIEmbedding(api_key=openai_api_key),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Simple RAG Example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_engine = index.as_query_engine(similarity_top_k=15)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "***********New LlamaParse+ Basic Query Engine***********\n",
+      "Multi-Head Attention is also known as multi-headed self-attention.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query = \"What is Multi-Head Attention also known as?\"\n",
+    "\n",
+    "response_1 = query_engine.query(query)\n",
+    "print(\"\\n***********New LlamaParse+ Basic Query Engine***********\")\n",
+    "print(response_1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'We used beam search as described in the previous section, but no\\ncheckpoint averaging. We present these results in Table 3.\\nIn Table 3 rows (A), we vary the number of attention heads and the attention key and value dimensions,\\nkeeping the amount of computation constant, as described in Section 3.2.2. While single-head\\nattention is 0.9 BLEU worse than the best setting, quality also drops off with too many heads.\\nIn Table 3 rows (B), we observe that reducing the attention key size dk hurts model quality. This\\nsuggests that determining compatibility is not easy and that a more sophisticated compatibility\\nfunction than dot product may be beneficial. We further observe in rows (C) and (D) that, as expected,\\nbigger models are better, and dropout is very helpful in avoiding over-fitting. In row (E) we replace our\\nsinusoidal positional encoding with learned positional embeddings [9], and observe nearly identical\\nresults to the base model.\\n6.3    English Constituency Parsing\\nTo evaluate if the Transformer can generalize to other tasks we performed experiments on English\\nconstituency parsing. This task presents specific challenges: the output is subject to strong structural\\nconstraints and is significantly longer than the input. Furthermore, RNN sequence-to-sequence\\nmodels have not been able to attain state-of-the-art results in small-data regimes [37].\\nWe trained a 4-layer transformer with dmodel = 1024 on the Wall Street Journal (WSJ) portion of the\\nPenn Treebank [25], about 40K training sentences. We also trained it in a semi-supervised setting,\\nusing the larger high-confidence and BerkleyParser corpora from with approximately 17M sentences\\n[37]. We used a vocabulary of 16K tokens for the WSJ only setting and a vocabulary of 32K tokens\\nfor the semi-supervised setting.\\nWe performed only a small number of experiments to select the dropout, both attention and residual\\n(section 5.4), learning rates and beam size on the Section 22 development set, all other parameters\\nremained unchanged from the English-to-German base translation model. During inference, we\\n                                                             9\\n---\\nTable 4: The Transformer generalizes well to English constituency parsing (Results are on Section 23\\nof WSJ)\\n                          Parser                           Training             WSJ 23 F1\\n            Vinyals & Kaiser el al. (2014) [37]    WSJ only, discriminative         88.3\\n                 Petrov et al. (2006) [29]         WSJ only, discriminative         90.4\\n                   Zhu et al. (2013) [40]          WSJ only, discriminative         90.4\\n                   Dyer et al. (2016) [8]          WSJ only, discriminative         91.7\\n                  Transformer (4 layers)           WSJ only, discriminative         91.3\\n                   Zhu et al. (2013) [40]              semi-supervised              91.3\\n               Huang & Harper (2009) [14]              semi-supervised              91.3\\n               McClosky et al. (2006) [26]             semi-supervised              92.1\\n            Vinyals & Kaiser el al. (2014) [37]        semi-supervised              92.1\\n                  Transformer (4 layers)               semi-supervised              92.7\\n                 Luong et al. (2015) [23]                  multi-task               93.0\\n                   Dyer et al. (2016) [8]                  generative               93.3\\nincreased the maximum output length to input length + 300. We used a beam size of 21 and α = 0.3\\nfor both WSJ only and the semi-supervised setting.\\nOur results in Table 4 show that despite the lack of task-specific tuning our model performs sur-\\nprisingly well, yielding better results than all previously reported models with the exception of the\\nRecurrent Neural Network Grammar [8].\\nIn contrast to RNN sequence-to-sequence models [37], the Transformer outperforms the Berkeley-\\nParser [29] even when training only on the WSJ training set of 40K sentences.\\n7    Conclusion\\nIn this work, we presented the Transformer, the first sequence transduction model based entirely on\\nattention, replacing the recurrent layers most commonly used in encoder-decoder architectures with\\nmulti-headed self-attention.\\nFor translation tasks, the Transformer can be trained significantly faster than architectures based\\non recurrent or convolutional layers.'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Take a look at one of the source nodes from the response\n",
+    "response_1.source_nodes[0].get_content()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/demo_table_comparisons.ipynb

@@ -0,0 +1,391 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# RAG for Table Comparisons with LlamaParse + LlamaIndex\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_table_comparisons.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "This notebook shows you how to do comparisons across both tabular and text data across multiple PDF documents.\n",
+    "\n",
+    "We load in multiple PDFs with embedded tables (2021 and 2020 10K filings for Apple) using LlamaParse, parse each into a hierarchy of tables/text objects, define a recursive retriever over each, and then compose both with a SubQuestionQueryEngine."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "Install core packages, download files, parse documents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install llama-index\n",
+    "!pip install llama-index-core\n",
+    "!pip install llama-index-embeddings-openai\n",
+    "!pip install llama-index-question-gen-openai\n",
+    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "!pip install llama-parse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!wget \"https://s2.q4cdn.com/470004039/files/doc_financials/2020/ar/_10-K-2020-(As-Filed).pdf\" -O apple_2020_10k.pdf\n",
+    "!wget \"https://s2.q4cdn.com/470004039/files/doc_financials/2021/q4/_10-K-2021-(As-Filed).pdf\" -O apple_2021_10k.pdf"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Some OpenAI and LlamaParse details"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "import os\n",
+    "# API access to llama-cloud\n",
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
+    "\n",
+    "# Using OpenAI API for embeddings/llms\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
+    "\n",
+    "Settings.llm = llm\n",
+    "Settings.embed_model = embed_model\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Using brand new `LlamaParse` PDF reader for PDF Parsing\n",
+    "\n",
+    "we also compare two different retrieval/query engine strategies:\n",
+    "1. Using raw Markdown text as nodes for building index and apply simple query engine for generating the results;\n",
+    "2. Using `MarkdownElementNodeParser` for parsing the `LlamaParse` output Markdown results and building recursive retriever query engine for generation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_parse import LlamaParse\n",
+    "\n",
+    "docs_2021 = LlamaParse(result_type=\"markdown\").load_data('./apple_2021_10k.pdf')\n",
+    "docs_2020 = LlamaParse(result_type=\"markdown\").load_data('./apple_2020_10k.pdf')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Recursive Retriever over each Document\n",
+    "\n",
+    "We define a function to get a recursive retriever from each document. The steps are the following:\n",
+    "- Hierarchically parse the document using our `MarkdownElementNodeParser`, which will embed/summarize embedded tables.\n",
+    "- Load into a vector store. Under the hood we will automatically store links between nodes (e.g. table summary to table text).\n",
+    "- Get a query engine over the vector store, which performs retrieval/synthesis. Under the hood we will automatically perform recursive retrieval if there are links."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
+    "\n",
+    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import pickle\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "\n",
+    "reranker = FlagEmbeddingReranker(\n",
+    "    top_n=5,\n",
+    "    model=\"BAAI/bge-reranker-large\",\n",
+    ")\n",
+    "\n",
+    "def create_query_engine_over_doc(docs, nodes_save_path=None):\n",
+    "    \"\"\"Big function to go from document path -> recursive retriever.\"\"\"\n",
+    "    if nodes_save_path is not None and os.path.exists(nodes_save_path):\n",
+    "        raw_nodes = pickle.load(open(nodes_save_path, \"rb\"))\n",
+    "    else:\n",
+    "        raw_nodes = node_parser.get_nodes_from_documents(docs)\n",
+    "        if nodes_save_path is not None:\n",
+    "            pickle.dump(raw_nodes, open(nodes_save_path, \"wb\"))\n",
+    "\n",
+    "    base_nodes, objects = node_parser.get_nodes_and_objects(\n",
+    "        raw_nodes\n",
+    "    )\n",
+    "\n",
+    "    ### Construct Retrievers\n",
+    "    # construct top-level vector index + query engine\n",
+    "    vector_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "    query_engine = vector_index.as_query_engine(\n",
+    "        similarity_top_k=15,\n",
+    "        node_postprocessors=[reranker]\n",
+    "    )\n",
+    "    return query_engine, base_nodes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "query_engine_2021, nodes_2021 = create_query_engine_over_doc(\n",
+    "    docs_2021, nodes_save_path=\"2021_nodes.pkl\"\n",
+    ")\n",
+    "query_engine_2020, nodes_2020 = create_query_engine_over_doc(\n",
+    "    docs_2020, nodes_save_path=\"2020_nodes.pkl\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
+    "from llama_index.core.query_engine import SubQuestionQueryEngine\n",
+    "\n",
+    "\n",
+    "# setup base query engine as tool\n",
+    "query_engine_tools = [\n",
+    "    QueryEngineTool(\n",
+    "        query_engine=query_engine_2021,\n",
+    "        metadata=ToolMetadata(\n",
+    "            name=\"apple_2021_10k\",\n",
+    "            description=(\n",
+    "                \"Provides information about Apple financials for year 2021\"\n",
+    "            ),\n",
+    "        ),\n",
+    "    ),\n",
+    "    QueryEngineTool(\n",
+    "        query_engine=query_engine_2020,\n",
+    "        metadata=ToolMetadata(\n",
+    "            name=\"apple_2020_10k\",\n",
+    "            description=(\n",
+    "                \"Provides information about Apple financials for year 2020\"\n",
+    "            ),\n",
+    "        ),\n",
+    "    ),\n",
+    "]\n",
+    "\n",
+    "sub_query_engine = SubQuestionQueryEngine.from_defaults(\n",
+    "    query_engine_tools=query_engine_tools,\n",
+    "    llm=llm,\n",
+    "    use_async=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Try out Some Comparisons"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generated 4 sub questions.\n",
+      "\u001b[1;3;38;2;237;90;200m[apple_2021_10k] Q: What are the deferred assets in 2021?\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2021_10k] Q: What are the deferred liabilities in 2021?\n",
+      "\u001b[0m\u001b[1;3;38;2;11;159;203m[apple_2020_10k] Q: What are the deferred assets in 2020?\n",
+      "\u001b[0m\u001b[1;3;38;2;155;135;227m[apple_2020_10k] Q: What are the deferred liabilities in 2020?\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2021_10k] A: $7,200\n",
+      "\u001b[0m\u001b[1;3;38;2;155;135;227m[apple_2020_10k] A: $10,138\n",
+      "\u001b[0m\u001b[1;3;38;2;237;90;200m[apple_2021_10k] A: $25,176 million\n",
+      "\u001b[0m\u001b[1;3;38;2;11;159;203m[apple_2020_10k] A: $19,336\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "response = sub_query_engine.query(\n",
+    "    \"Can you compare and contrast the deferred assets and liabilities in 2021 with 2020?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "In 2021, the deferred assets increased by $5,840 million compared to 2020, while the deferred liabilities decreased by $2,938 million in the same period.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(str(response))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generated 2 sub questions.\n",
+      "\u001b[1;3;38;2;237;90;200m[apple_2021_10k] Q: What is the total number of RSUs in Apple's 2021 financials?\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2020_10k] Q: What is the total number of RSUs in Apple's 2020 financials?\n",
+      "\u001b[0m\u001b[1;3;38;2;237;90;200m[apple_2021_10k] A: The total number of RSUs in Apple's 2021 financials is 240,427.\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2020_10k] A: The total number of RSUs in Apple's 2020 financials is 310,778.\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "response = sub_query_engine.query(\n",
+    "    \"Can you compare and contrast the total number of RSUs in 2021 and 2020?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generated 2 sub questions.\n",
+      "\u001b[1;3;38;2;237;90;200m[apple_2021_10k] Q: What are the risk factors mentioned in the 2021 financial report of Apple?\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2020_10k] Q: What are the risk factors mentioned in the 2020 financial report of Apple?\n",
+      "\u001b[0m\u001b[1;3;38;2;237;90;200m[apple_2021_10k] A: The risk factors mentioned in the 2021 financial report of Apple include risks related to COVID-19, macroeconomic and industry risks, political events, trade and international disputes, natural disasters, public health issues, industrial accidents, credit risk, fluctuations in foreign currency exchange rates, changes in tax rates and legislation, volatility in the price of the company's stock, and exposure to legal proceedings and claims.\n",
+      "\u001b[0m\u001b[1;3;38;2;90;149;237m[apple_2020_10k] A: The risk factors mentioned in the 2020 financial report of Apple include the impact of the COVID-19 pandemic on the company's business operations, financial condition, and stock price; global and regional economic conditions affecting demand for products and services; competition in global markets with rapid technological changes; potential disruptions in the supply chain due to industrial accidents or public health issues; information technology system failures or network disruptions affecting business operations; risks associated with confidential information security and potential unauthorized access; fluctuations in quarterly net sales and operating results due to various factors; stock price volatility impacting investor confidence and employee retention; financial performance risks related to changes in foreign currency exchange rates affecting sales and earnings.\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "response = sub_query_engine.query(\n",
+    "    \"Can you compare and contrast the risk factors in 2021 vs. 2020?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The risk factors mentioned in the 2021 financial report of Apple include risks related to COVID-19, macroeconomic and industry risks, political events, trade and international disputes, natural disasters, public health issues, industrial accidents, credit risk, fluctuations in foreign currency exchange rates, changes in tax rates and legislation, volatility in the price of the company's stock, and exposure to legal proceedings and claims. In contrast, the risk factors mentioned in the 2020 financial report of Apple focused more on the impact of the COVID-19 pandemic on the company's business operations, financial condition, and stock price; global and regional economic conditions affecting demand for products and services; competition in global markets with rapid technological changes; potential disruptions in the supply chain due to industrial accidents or public health issues; information technology system failures or network disruptions affecting business operations; risks associated with confidential information security and potential unauthorized access; fluctuations in quarterly net sales and operating results due to various factors; stock price volatility impacting investor confidence and employee retention; financial performance risks related to changes in foreign currency exchange rates affecting sales and earnings.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(str(response))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "llama_parse",
+   "language": "python",
+   "name": "llama_parse"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

llama_parse/base.py

@@ -4,6 +4,7 @@ import httpx
 import mimetypes
 import time
 from enum import Enum
+from pathlib import Path
 from typing import List, Optional, Union
 
 from llama_index.core.async_utils import run_jobs
@@ -21,6 +22,91 @@ class ResultType(str, Enum):
     TXT = "text"
     MD = "markdown"
 
+class Language(str, Enum):
+    BAZA = "abq"
+    ADYGHE = "ady"
+    AFRIKAANS = "af"
+    ANGIKA = "ang"
+    ARABIC = "ar"
+    ASSAMESE = "as"
+    AVAR = "ava"
+    AZERBAIJANI = "az"
+    BELARUSIAN = "be"
+    BULGARIAN = "bg"
+    BIHARI = "bh"
+    BHOJPURI = "bho"
+    BENGALI = "bn"
+    BOSNIAN = "bs"
+    SIMPLIFIED_CHINESE = "ch_sim"
+    TRADITIONAL_CHINESE = "ch_tra"
+    CHECHEN = "che"
+    CZECH = "cs"
+    WELSH = "cy"
+    DANISH = "da"
+    DARGWA = "dar"
+    GERMAN = "de"
+    ENGLISH = "en"
+    SPANISH = "es"
+    ESTONIAN = "et"
+    PERSIAN_FARSI = "fa"
+    FRENCH = "fr"
+    IRISH = "ga"
+    GOAN_KONKANI = "gom"
+    HINDI = "hi"
+    CROATIAN = "hr"
+    HUNGARIAN = "hu"
+    INDONESIAN = "id"
+    INGUSH = "inh"
+    ICELANDIC = "is"
+    ITALIAN = "it"
+    JAPANESE = "ja"
+    KABARDIAN = "kbd"
+    KANNADA = "kn"
+    KOREAN = "ko"
+    KURDISH = "ku"
+    LATIN = "la"
+    LAK = "lbe"
+    LEZGHIAN = "lez"
+    LITHUANIAN = "lt"
+    LATVIAN = "lv"
+    MAGAHI = "mah"
+    MAITHILI = "mai"
+    MAORI = "mi"
+    MONGOLIAN = "mn"
+    MARATHI = "mr"
+    MALAY = "ms"
+    MALTESE = "mt"
+    NEPALI = "ne"
+    NEWARI = "new"
+    DUTCH = "nl"
+    NORWEGIAN = "no"
+    OCCITAN = "oc"
+    PALI = "pi"
+    POLISH = "pl"
+    PORTUGUESE = "pt"
+    ROMANIAN = "ro"
+    RUSSIAN = "ru"
+    SERBIAN_CYRILLIC = "rs_cyrillic"
+    SERBIAN_LATIN = "rs_latin"
+    NAGPURI = "sck"
+    SLOVAK = "sk"
+    SLOVENIAN = "sl"
+    ALBANIAN = "sq"
+    SWEDISH = "sv"
+    SWAHILI = "sw"
+    TAMIL = "ta"
+    TABASSARAN = "tab"
+    TELUGU = "te"
+    THAI = "th"
+    TAJIK = "tjk"
+    TAGALOG = "tl"
+    TURKISH = "tr"
+    UYGHUR = "ug"
+    UKRANIAN = "uk"
+    URDU = "ur"
+    UZBEK = "uz"
+    VIETNAMESE = "vi"
+
 
 class LlamaParse(BasePydanticReader):
     """A smart-parser for files."""
@@ -50,6 +136,9 @@ class LlamaParse(BasePydanticReader):
     verbose: bool = Field(
         default=True, description="Whether to print the progress of the parsing."
     )
+    language: Optional[str] = Field(
+         default=Language.ENGLISH, description="The language of the text to parse."
+    )
 
     @validator("api_key", pre=True, always=True)
     def validate_api_key(cls, v: str) -> str:
@@ -89,7 +178,7 @@ class LlamaParse(BasePydanticReader):
                 # send the request, start job
                 url = f"{self.base_url}/api/parsing/upload"
                 async with httpx.AsyncClient(timeout=self.max_timeout) as client:
-                    response = await client.post(url, files=files, headers=headers)
+                    response = await client.post(url, files=files, headers=headers, data={"language": self.language})
                     if not response.is_success:
                         raise Exception(f"Failed to parse the PDF file: {response.text}")
 
@@ -135,7 +224,7 @@ class LlamaParse(BasePydanticReader):
     
     async def aload_data(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[Document]:
         """Load data from the input path."""
-        if isinstance(file_path, str):
+        if isinstance(file_path, (str, Path)):
             return await self._aload_data(file_path, extra_info=extra_info)
         elif isinstance(file_path, list):
             jobs = [self._aload_data(f, extra_info=extra_info) for f in file_path]
@@ -160,4 +249,4 @@ class LlamaParse(BasePydanticReader):
             if nest_asyncio_err in str(e):
                 raise RuntimeError(nest_asyncio_msg)
             else:
-                raise e
+                raise e

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "llama-parse"
-version = "0.3.3"
+version = "0.3.5"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"