Ensure image extensions, vbump (#159 )

add html support (#154 )
some notebook updates (#148 )
2026-07-20 19:47:38 -04:00 · 2024-04-24 20:39:39 -06:00 · 2024-04-23 13:14:13 -06:00 · 2024-04-18 21:57:10 -06:00 · 2024-04-14 15:17:31 -06:00 · 2024-04-14 15:12:22 -06:00
30 changed files with 3551 additions and 1050 deletions
@@ -0,0 +1,48 @@
+name: Build Package
+
+# Build package on its own without additional pip install
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: poetry install
+      - name: Ensure lock works
+        shell: bash
+        run: poetry lock
+      - name: Build
+        shell: bash
+        run: poetry build
+      - name: Test installing built package
+        shell: bash
+        run: python -m pip install .
+      - name: Test import
+        shell: bash
+        working-directory: ${{ vars.RUNNER_TEMP }}
+        run: python -c "import llama_parse"
@@ -0,0 +1,81 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: ["main"]
+  schedule:
+    - cron: "30 16 * * 4"
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["python"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+      # - run: |
+      #     echo "Run, Build Application using script"
+      #     ./location_of_script_within_repo/buildscript.sh
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{matrix.language}}"
@@ -0,0 +1,37 @@
+name: Linting
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install pre-commit
+        shell: bash
+        run: poetry run pip install pre-commit
+      - name: Run linter
+        shell: bash
+        run: poetry run make lint
@@ -0,0 +1,64 @@
+name: Publish llama-parse to PyPI / GitHub
+
+on:
+  push:
+    tags:
+      - "v*"
+
+  workflow_dispatch:
+
+env:
+  POETRY_VERSION: "1.6.1"
+  PYTHON_VERSION: "3.9"
+
+jobs:
+  build-n-publish:
+    name: Build and publish to PyPI
+    if: github.repository == 'run-llama/llama_parse'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: pip install -e .
+      - name: Build and publish to pypi
+        uses: JRubics/poetry-publish@v1.17
+        with:
+          pypi_token: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
+          ignore_dev_requirements: "yes"
+
+      - name: Create GitHub Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Get Asset name
+        run: |
+          export PKG=$(ls dist/ | grep tar)
+          set -- $PKG
+          echo "name=$1" >> $GITHUB_ENV
+      - name: Upload Release Asset (sdist) to GitHub
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: dist/${{ env.name }}
+          asset_name: ${{ env.name }}
+          asset_content_type: application/zip
@@ -0,0 +1,40 @@
+name: Unit Testing
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+  LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        python-version: ["3.8", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: poetry install --with dev
+      - name: Run testing
+        env:
+          CI: true
+        shell: bash
+        run: poetry run pytest tests
@@ -1,4 +1,4 @@
 .git
 __pycache__/
 *.pyc
-.DS_Store
+.DS_Store
@@ -0,0 +1,86 @@
+---
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-byte-order-marker
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.1.5
+
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        exclude: ".*poetry.lock"
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.10.1
+    hooks:
+      - id: black-jupyter
+        name: black-src
+        alias: black
+        exclude: ".*poetry.lock"
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.0.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [
+            "types-requests",
+            "types-Deprecated",
+            "types-redis",
+            "types-setuptools",
+            "types-PyYAML",
+            "types-protobuf==4.24.0.4",
+          ]
+        args:
+          [
+            --disallow-untyped-defs,
+            --ignore-missing-imports,
+            --python-version=3.8,
+          ]
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+      - id: blacken-docs
+        name: black-docs-text
+        alias: black
+        types_or: [rst, markdown, tex]
+        additional_dependencies: [black==23.10.1]
+        # Using PEP 8's line length in docs prevents excess left/right scrolling
+        args: [--line-length=79]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier
+        exclude: poetry.lock
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies: [tomli]
+        exclude: ^(poetry.lock|examples)
+        args:
+          [
+            "--ignore-words-list",
+            "astroid,gallary,momento,narl,ot,rouge,nin,gere,te,inh",
+          ]
+  - repo: https://github.com/srstevenson/nb-clean
+    rev: 3.1.0
+    hooks:
+      - id: nb-clean
+        args: [--preserve-cell-outputs, --remove-empty-cells]
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
+        exclude: ".*poetry.lock"
@@ -0,0 +1,14 @@
+GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
+
+help:	## Show all Makefile targets.
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+
+format:	## Run code autoformatters (black).
+	pre-commit install
+	git ls-files | xargs pre-commit run black --files
+
+lint:	## Run linters: pre-commit (black, ruff, codespell) and mypy
+	pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+
+test:	## Run tests via pytest
+	pytest tests
@@ -4,12 +4,11 @@ LlamaParse is an API created by LlamaIndex to efficiently parse and represent fi

 LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).

-
 Free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page.

 ## Getting Started

-First, login and get an api-key from `https://cloud.llamaindex.ai`.
+First, login and get an api-key from [**https://cloud.llamaindex.ai ↗**](https://cloud.llamaindex.ai).

 Then, make sure you have the latest LlamaIndex version installed.

@@ -28,6 +27,7 @@ Now you can run the following to parse your first PDF file:

 ```python
 import nest_asyncio
+
 nest_asyncio.apply()

 from llama_parse import LlamaParse
@@ -35,9 +35,9 @@ from llama_parse import LlamaParse
 parser = LlamaParse(
    api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
    result_type="markdown",  # "markdown" and "text" are available
-    num_workers=4, # if multiple files passed, split in `num_workers` API calls
+    num_workers=4,  # if multiple files passed, split in `num_workers` API calls
    verbose=True,
-    language="en" # Optionaly you can define a language, default=en
+    language="en",  # Optionally you can define a language, default=en
 )

 # sync
@@ -59,6 +59,7 @@ You can also integrate the parser as the default PDF loader in `SimpleDirectoryR

 ```python
 import nest_asyncio
+
 nest_asyncio.apply()

 from llama_parse import LlamaParse
@@ -67,11 +68,13 @@ from llama_index.core import SimpleDirectoryReader
 parser = LlamaParse(
    api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
    result_type="markdown",  # "markdown" and "text" are available
-    verbose=True
+    verbose=True,
 )

 file_extractor = {".pdf": parser}
-documents = SimpleDirectoryReader("./data", file_extractor=file_extractor).load_data()
+documents = SimpleDirectoryReader(
+    "./data", file_extractor=file_extractor
+).load_data()
 ```

 Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex Documentation](https://docs.llamaindex.ai/en/stable/module_guides/loading/simpledirectoryreader.html).
@@ -0,0 +1,302 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LlamaParse Agent\n",
+    "\n",
+    "This demo walks through using an OpenAI Agent with [LlamaParse](https://cloud.llamaindex.ai)."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install llama-parse llama-index llama-index-postprocessor-sbert-rerank"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import Settings\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "\n",
+    "Settings.embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "Settings.llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Parsing \n",
+    "\n",
+    "For parsing, lets use a [recent paper](https://huggingface.co/papers/2403.09611) on Multi-Modal pretraining"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!wget https://arxiv.org/pdf/2403.09611.pdf -O paper.pdf"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Below, we can tell the parser to skip content we don't want. In this case, the references section will just add noise to a RAG system."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_parse import LlamaParse\n",
+    "\n",
+    "parser = LlamaParse(\n",
+    "    result_type=\"markdown\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 81251f39-01be-434e-99e8-1c1b83b82098\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = await parser.aload_data(\"paper.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Embeddings have been explicitly disabled. Using MockEmbedding.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "41it [00:00, 26765.21it/s]\n",
+      "100%|██████████| 41/41 [00:13<00:00,  2.98it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "from llama_index.core.node_parser import (\n",
+    "    MarkdownElementNodeParser,\n",
+    "    SentenceSplitter,\n",
+    ")\n",
+    "\n",
+    "# explicitly extract tables with the MarkdownElementNodeParser\n",
+    "node_parser = MarkdownElementNodeParser(num_workers=8)\n",
+    "nodes = node_parser.get_nodes_from_documents(documents)\n",
+    "nodes, objects = node_parser.get_nodes_and_objects(nodes)\n",
+    "\n",
+    "# Chain splitters to ensure chunk size requirements are met\n",
+    "nodes = SentenceSplitter(chunk_size=512, chunk_overlap=20).get_nodes_from_documents(\n",
+    "    nodes\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Chat over the paper, lets find out what it is about!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import VectorStoreIndex, SummaryIndex\n",
+    "\n",
+    "vector_index = VectorStoreIndex(nodes=nodes)\n",
+    "summary_index = SummaryIndex(nodes=nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.agent.openai import OpenAIAgent\n",
+    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
+    "from llama_index.postprocessor.colbert_rerank import ColbertRerank\n",
+    "\n",
+    "tools = [\n",
+    "    QueryEngineTool(\n",
+    "        vector_index.as_query_engine(\n",
+    "            similarity_top_k=8, node_postprocessors=[ColbertRerank(top_n=3)]\n",
+    "        ),\n",
+    "        metadata=ToolMetadata(\n",
+    "            name=\"search\",\n",
+    "            description=\"Search the document, pass the entire user message in the query\",\n",
+    "        ),\n",
+    "    ),\n",
+    "    QueryEngineTool(\n",
+    "        summary_index.as_query_engine(),\n",
+    "        metadata=ToolMetadata(\n",
+    "            name=\"summarize\",\n",
+    "            description=\"Summarize the document using the user message\",\n",
+    "        ),\n",
+    "    ),\n",
+    "]\n",
+    "\n",
+    "agent = OpenAIAgent.from_tools(tools=tools, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Added user message to memory: What is the summary of the paper?\n",
+      "=== Calling Function ===\n",
+      "Calling function: summarize with args: {\"input\":\"summary\"}\n",
+      "Got output: The research focuses on developing Multimodal Large Language Models (MLLMs) by incorporating image-caption, interleaved image-text, and text-only data for pre-training. It highlights the importance of factors like the image encoder, resolution, and token count, while downplaying the design of the vision-language connector. With models scaling up to 30B parameters, the MM1 family demonstrates impressive performance in pre-training metrics and competitive outcomes on diverse multimodal benchmarks. It demonstrates abilities such as in-context learning and multi-image reasoning, aiming to provide valuable insights for creating MLLMs that benefit the research community.\n",
+      "========================\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# note -- this will take a while with local LLMs, its sending every node in the document to the LLM\n",
+    "resp = agent.chat(\"What is the summary of the paper?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The summary of the paper highlights the development of Multimodal Large Language Models (MLLMs) by incorporating image-caption, interleaved image-text, and text-only data for pre-training. The research emphasizes factors like the image encoder, resolution, and token count, while de-emphasizing the design of the vision-language connector. The MM1 family of models, scaling up to 30B parameters, shows impressive performance in pre-training metrics and competitive outcomes on various multimodal benchmarks. These models demonstrate capabilities such as in-context learning and multi-image reasoning, aiming to provide valuable insights for creating MLLMs that benefit the research community.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(str(resp))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Added user message to memory: How do the authors evaluate their work?\n",
+      "=== Calling Function ===\n",
+      "Calling function: search with args: {\"input\":\"evaluation methods\"}\n",
+      "Got output: The evaluation methods involve synthesizing all benchmark results into a single meta-average number to simplify comparisons. This is achieved by normalizing the evaluation metrics with respect to a baseline configuration, standardizing the results for each task, adjusting every metric by dividing it by its respective baseline, and then averaging across all metrics.\n",
+      "========================\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "resp = agent.chat(\"How do the authors evaluate their work?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The authors evaluate their work by synthesizing all benchmark results into a single meta-average number to simplify comparisons. They normalize the evaluation metrics with respect to a baseline configuration, standardize the results for each task, adjust every metric by dividing it by its respective baseline, and then average across all metrics for evaluation.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(str(resp))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "llama-parse-aNC435Vv-py3.10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
@@ -4,9 +4,11 @@
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "# Llama Parser <> LlamaIndex\n",
+    "# Advanced RAG with LlamaParse\n",
    "\n",
-    "This notebook is a complete walkthrough for using `LlamaParse` for RAG applications with `LlamaIndex`.\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "This notebook shows you how to use LlamaParse with our advanced markdown ingestion and recursive retrieval algorithms to model tables/text within a document hierarchically. This lets you ask questions over both tables and text.\n",
    "\n",
    "Note for this example, we are using the `llama_index >=0.10.4` version"
   ]
@@ -17,12 +19,12 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
   ]
  },
  {
@@ -31,7 +33,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -43,15 +45,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -61,7 +65,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -70,11 +74,11 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
-    "Settings.embed_model = embed_model\n"
+    "Settings.embed_model = embed_model"
   ]
  },
  {
@@ -90,7 +94,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -104,12 +108,12 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "documents = LlamaParse(result_type=\"markdown\").load_data('./uber_10q_march_2022.pdf')"
+    "documents = LlamaParse(result_type=\"markdown\").load_data(\"./uber_10q_march_2022.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -144,23 +148,25 @@
    }
   ],
   "source": [
-    "print(documents[0].text[:1000] + '...')"
+    "print(documents[0].text[:1000] + \"...\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -185,7 +191,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -194,21 +200,23 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "recursive_index = VectorStoreIndex(nodes=base_nodes + objects)\n",
    "raw_index = VectorStoreIndex.from_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
@@ -216,17 +224,17 @@
    ")\n",
    "\n",
    "recursive_query_engine = recursive_index.as_query_engine(\n",
-    "    similarity_top_k=15, \n",
-    "    node_postprocessors=[reranker], \n",
-    "    verbose=True\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker], verbose=True\n",
    ")\n",
    "\n",
-    "raw_query_engine = raw_index.as_query_engine(similarity_top_k=15, node_postprocessors=[reranker])"
+    "raw_query_engine = raw_index.as_query_engine(\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker]\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -258,7 +266,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -295,7 +303,7 @@
    "\n",
    "response_2 = recursive_query_engine.query(query)\n",
    "print(\"\\n***********New LlamaParse+ Recursive Retriever Query Engine***********\")\n",
-    "print(response_2)\n"
+    "print(response_2)"
   ]
  },
  {
@@ -312,7 +320,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -372,7 +380,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -428,7 +436,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -502,8 +510,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -17,13 +17,13 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core==0.10.6.post1\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse\n",
-    "!pip install llama-index-vector-stores-astra-db"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core==0.10.6.post1\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse\n",
+    "%pip install llama-index-vector-stores-astra-db"
   ]
  },
  {
@@ -32,7 +32,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -44,15 +44,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -66,7 +68,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -75,7 +77,7 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
@@ -93,7 +95,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -107,12 +109,12 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "documents = LlamaParse(result_type=\"markdown\").load_data('./uber_10q_march_2022.pdf')"
+    "documents = LlamaParse(result_type=\"markdown\").load_data(\"./uber_10q_march_2022.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -168,7 +170,7 @@
    }
   ],
   "source": [
-    "print(documents[0].text[:1000] + '...')"
+    "print(documents[0].text[:1000] + \"...\")"
   ]
  },
  {
@@ -180,29 +182,27 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.vector_stores.astra_db import AstraDBVectorStore\n",
    "\n",
-    "# define two storage classes representing two collections (to compare advanced approach vs. baseline) \n",
+    "# define two storage classes representing two collections (to compare advanced approach vs. baseline)\n",
    "\n",
    "astra_db_store_advanced = AstraDBVectorStore(\n",
    "    token=ASTRA_TOKEN,\n",
    "    api_endpoint=ASTRA_API_ENDPOINT,\n",
    "    namespace=ASTRA_NAMESPACE,\n",
    "    collection_name=\"astra_v_table_llamaparse_advanced\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")\n",
    "astra_db_store_base = AstraDBVectorStore(\n",
    "    token=ASTRA_TOKEN,\n",
    "    api_endpoint=ASTRA_API_ENDPOINT,\n",
    "    namespace=ASTRA_NAMESPACE,\n",
    "    collection_name=\"astra_v_table_llamaparse_base\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")"
   ]
  },
@@ -219,13 +219,15 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
@@ -239,7 +241,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -248,17 +250,23 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import StorageContext\n",
    "\n",
-    "storage_context_advanced = StorageContext.from_defaults(vector_store=astra_db_store_advanced)\n",
+    "storage_context_advanced = StorageContext.from_defaults(\n",
+    "    vector_store=astra_db_store_advanced\n",
+    ")\n",
    "storage_context_base = StorageContext.from_defaults(vector_store=astra_db_store_base)\n",
    "\n",
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects, storage_context=storage_context_advanced)\n",
-    "raw_index = VectorStoreIndex.from_documents(documents, storage_context=storage_context_base)"
+    "recursive_index = VectorStoreIndex(\n",
+    "    nodes=base_nodes + objects, storage_context=storage_context_advanced\n",
+    ")\n",
+    "raw_index = VectorStoreIndex.from_documents(\n",
+    "    documents, storage_context=storage_context_base\n",
+    ")"
   ]
  },
  {
@@ -267,7 +275,9 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
@@ -275,12 +285,12 @@
    ")\n",
    "\n",
    "recursive_query_engine = recursive_index.as_query_engine(\n",
-    "    similarity_top_k=15, \n",
-    "    node_postprocessors=[reranker], \n",
-    "    verbose=True\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker], verbose=True\n",
    ")\n",
    "\n",
-    "raw_query_engine = raw_index.as_query_engine(similarity_top_k=15, node_postprocessors=[reranker])"
+    "raw_query_engine = raw_index.as_query_engine(\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker]\n",
+    ")"
   ]
  },
  {
@@ -300,7 +310,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -341,7 +351,7 @@
    "\n",
    "response_2 = recursive_query_engine.query(query)\n",
    "print(\"\\n***********New LlamaParse+ Recursive Retriever Query Engine***********\")\n",
-    "print(response_2)\n"
+    "print(response_2)"
   ]
  },
  {
@@ -358,7 +368,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -422,7 +432,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -478,7 +488,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -554,8 +564,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -11,7 +11,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -38,7 +38,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -47,7 +47,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -88,7 +88,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -128,10 +128,8 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  },
-  "orig_nbformat": 4
+   "pygments_lexer": "ipython3"
+  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
@@ -23,12 +23,12 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, install the required dependencies\n",
-    "!pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
+    "%pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
   ]
  },
  {
@@ -40,7 +40,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -53,7 +53,9 @@
    "llama_cloud_api_key = getpass(\"Enter your Llama Index Cloud API Key: \")\n",
    "api_endpoint = input(\"Enter your Astra DB API Endpoint: \")\n",
    "token = getpass(\"Enter your Astra DB Token: \")\n",
-    "namespace = input(\"Enter your Astra DB namespace (optional, must exist on Astra): \") or None\n",
+    "namespace = (\n",
+    "    input(\"Enter your Astra DB namespace (optional, must exist on Astra): \") or None\n",
+    ")\n",
    "openai_api_key = getpass(\"Enter your OpenAI API Key: \")\n",
    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = llama_cloud_api_key\n",
@@ -62,7 +64,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -81,7 +83,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -94,7 +96,7 @@
   ],
   "source": [
    "# Grab a PDF from Arxiv for indexing\n",
-    "import requests \n",
+    "import requests\n",
    "\n",
    "# The URL of the file you want to download\n",
    "url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
@@ -116,7 +118,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -135,7 +137,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -144,7 +146,7 @@
       "'rmer - model architecture.\\nThe Transformer follows this overall architecture using stacked self-attention and point-wise, fully\\nconnected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\\nrespectively.\\n3.1   Encoder and Decoder Stacks\\nEncoder:     The encoder is composed of a stack of N = 6 identical layers. Each layer has two\\nsub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\\nwise fully connected feed-forward network. We employ a residual connection [11] around each of\\nthe two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\\nLayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\\nitself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\\nlayers, produce outputs of dimension dmodel = 512.\\nDecoder:    The decoder is also composed of a stack of N = 6 identical layers. In addition '"
      ]
     },
-     "execution_count": 6,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -163,7 +165,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -174,16 +176,14 @@
    "    api_endpoint=api_endpoint,\n",
    "    namespace=namespace,\n",
    "    collection_name=\"astra_v_table_llamaparse\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import SimpleNodeParser\n",
@@ -195,7 +195,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -220,7 +220,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -229,7 +229,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -252,7 +252,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -261,7 +261,7 @@
       "'We used beam search as described in the previous section, but no\\ncheckpoint averaging. We present these results in Table 3.\\nIn Table 3 rows (A), we vary the number of attention heads and the attention key and value dimensions,\\nkeeping the amount of computation constant, as described in Section 3.2.2. While single-head\\nattention is 0.9 BLEU worse than the best setting, quality also drops off with too many heads.\\nIn Table 3 rows (B), we observe that reducing the attention key size dk hurts model quality. This\\nsuggests that determining compatibility is not easy and that a more sophisticated compatibility\\nfunction than dot product may be beneficial. We further observe in rows (C) and (D) that, as expected,\\nbigger models are better, and dropout is very helpful in avoiding over-fitting. In row (E) we replace our\\nsinusoidal positional encoding with learned positional embeddings [9], and observe nearly identical\\nresults to the base model.\\n6.3    English Constituency Parsing\\nTo evaluate if the Transformer can generalize to other tasks we performed experiments on English\\nconstituency parsing. This task presents specific challenges: the output is subject to strong structural\\nconstraints and is significantly longer than the input. Furthermore, RNN sequence-to-sequence\\nmodels have not been able to attain state-of-the-art results in small-data regimes [37].\\nWe trained a 4-layer transformer with dmodel = 1024 on the Wall Street Journal (WSJ) portion of the\\nPenn Treebank [25], about 40K training sentences. We also trained it in a semi-supervised setting,\\nusing the larger high-confidence and BerkleyParser corpora from with approximately 17M sentences\\n[37]. We used a vocabulary of 16K tokens for the WSJ only setting and a vocabulary of 32K tokens\\nfor the semi-supervised setting.\\nWe performed only a small number of experiments to select the dropout, both attention and residual\\n(section 5.4), learning rates and beam size on the Section 22 development set, all other parameters\\nremained unchanged from the English-to-German base translation model. During inference, we\\n                                                             9\\n---\\nTable 4: The Transformer generalizes well to English constituency parsing (Results are on Section 23\\nof WSJ)\\n                          Parser                           Training             WSJ 23 F1\\n            Vinyals & Kaiser el al. (2014) [37]    WSJ only, discriminative         88.3\\n                 Petrov et al. (2006) [29]         WSJ only, discriminative         90.4\\n                   Zhu et al. (2013) [40]          WSJ only, discriminative         90.4\\n                   Dyer et al. (2016) [8]          WSJ only, discriminative         91.7\\n                  Transformer (4 layers)           WSJ only, discriminative         91.3\\n                   Zhu et al. (2013) [40]              semi-supervised              91.3\\n               Huang & Harper (2009) [14]              semi-supervised              91.3\\n               McClosky et al. (2006) [26]             semi-supervised              92.1\\n            Vinyals & Kaiser el al. (2014) [37]        semi-supervised              92.1\\n                  Transformer (4 layers)               semi-supervised              92.7\\n                 Luong et al. (2015) [23]                  multi-task               93.0\\n                   Dyer et al. (2016) [8]                  generative               93.3\\nincreased the maximum output length to input length + 300. We used a beam size of 21 and α = 0.3\\nfor both WSJ only and the semi-supervised setting.\\nOur results in Table 4 show that despite the lack of task-specific tuning our model performs sur-\\nprisingly well, yielding better results than all previously reported models with the exception of the\\nRecurrent Neural Network Grammar [8].\\nIn contrast to RNN sequence-to-sequence models [37], the Transformer outperforms the Berkeley-\\nParser [29] even when training only on the WSJ training set of 40K sentences.\\n7    Conclusion\\nIn this work, we presented the Transformer, the first sequence transduction model based entirely on\\nattention, replacing the recurrent layers most commonly used in encoder-decoder architectures with\\nmulti-headed self-attention.\\nFor translation tasks, the Transformer can be trained significantly faster than architectures based\\non recurrent or convolutional layers.'"
      ]
     },
-     "execution_count": 12,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -287,8 +287,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -13,12 +13,12 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -45,7 +45,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -55,12 +55,13 @@
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -79,7 +80,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -107,7 +108,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -126,7 +127,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -157,13 +158,6 @@
   "source": [
    "print(documents[0].text[20000:21000] + \"...\")"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -181,10 +175,8 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  },
-  "orig_nbformat": 4
+   "pygments_lexer": "ipython3"
+  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
@@ -0,0 +1,531 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LlamaParse - Fast checking Insurance Contract for Coverage\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_insurance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "In this notebook we will look at how LlamaParse can be used to extract structured coverage information from an insurance policy."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Installation of required packages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install llama-index llama-parse"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Download an insurance policy fron IRDAI\n",
+    "\n",
+    "The Insurance Regulatory and Development Authority of India (IRDAI) maintains a great resource: https://policyholder.gov.in/web/guest/non-life-insurance-products where all insurance policies available in India are publicly available for download! Let's download a complex health insurance policy as an example."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!wget \"https://policyholder.gov.in/documents/37343/931203/NBHTGBP22011V012223.pdf/c392bcc1-f6a8-cadd-ab84-495b3273d2c3?version=1.0&t=1669350459879&download=true\" -O \"./policy.pdf\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Initializing LlamaIndex and LlamaParse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "# for the purpose of this example, we will use the small model embedding and gpt3.5\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
+    "\n",
+    "Settings.llm = llm"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Vanilla Approach - Parse the Policy with LlamaParse into Markdown"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id b8946573-c911-4e00-8921-1bad1cda3d64\n",
+      "......"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_parse import LlamaParse\n",
+    "\n",
+    "documents = LlamaParse(result_type=\"markdown\").load_data(\"./policy.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "## Preamble\n",
+      "\n",
+      "This ‘Travel Infinity’ Policy is a contract of insurance between You and Us which is subject to payment of full premium in advance and the terms, conditions and exclusions of this Policy. Expense incurred outside the policy period will NOT be covered. Unutilized Sum Insured will expire at the end of the policy year. All applicable benefits, details and limits are mentioned in your Certificate of insurance. We will cover only allopathic treatments in this policy.\n",
+      "\n",
+      "## Defined Terms\n",
+      "\n",
+      "The terms listed below in this Section and used elsewhere in the Policy in Initial Capitals shall have the meaning set out against them in this Section.\n",
+      "\n",
+      "### Standard Definitions\n",
+      "\n",
+      "|2.1|Accident or Accidental|means sudden, unforeseen and involuntary event caused by external, visible and violent means.|\n",
+      "|---|---|---|\n",
+      "|2.2|Co-payment|means a cost sharing requirement under a health insurance policy that provides that the policyholder/insured will bear a specified percentage of the admissible claims a\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(documents[0].text[0:1000])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Markdown Element Node Parser\n",
+    "Our markdown element node parser works well for parsing the markdown output of LlamaParse into a set of table and text nodes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
+    "\n",
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "base_nodes, objects = node_parser.get_nodes_and_objects(nodes)\n",
+    "\n",
+    "recursive_index = VectorStoreIndex(nodes=base_nodes + objects)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_engine = recursive_index.as_query_engine(similarity_top_k=25)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Querying the model for coverage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "You are covered for the expenses incurred on any alternate travel booking under any mode of transport, up to the limit of the Sum Insured as mentioned in the Certificate of insurance, if the delay of the airlines was caused due to specific reasons outlined in the policy. The amount you are covered for will depend on the specific terms and conditions of your policy, including the maximum coverage limit specified in the Certificate of insurance.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_1 = \"My trip was delay and I paid 45, how much am I cover for?\"\n",
+    "\n",
+    "response_1 = query_engine.query(query_1)\n",
+    "print(str(response_1))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The information is split across the document which leads to retrieval issues. Let's try some parsing instructions to improve our result."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id ec9e77c9-6ad9-4c9b-9efb-c9f659b0d481\n",
+      "....."
+     ]
+    }
+   ],
+   "source": [
+    "documents_with_instruction = LlamaParse(\n",
+    "    result_type=\"markdown\",\n",
+    "    parsing_instruction=\"\"\"\n",
+    "This document is an insurance policy.\n",
+    "When a benefits/coverage/exlusion is describe in the document ammend to it add a text in the follwing benefits string format (where coverage could be an exclusion).\n",
+    "\n",
+    "For {nameofrisk} and in this condition {whenDoesThecoverageApply} the coverage is {coverageDescription}. \n",
+    "                                        \n",
+    "If the document contain a benefits TABLE that describe coverage amounts, do not ouput it as a table, but instead as a list of benefits string.\n",
+    "                                       \n",
+    "\"\"\",\n",
+    ").load_data(\"./policy.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let see how the 2 parsing compare (change target page to explore)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "## Inpatient treatment\n",
+      "\n",
+      "Claim Form (filled and signed by pe Insured)\n",
+      "Hospital Daily Cash\n",
+      "Release of Medical information Form (filled and signed by pe Insured)\n",
+      "Waiver of Deductible\n",
+      "Original papological and diagnostic reports, discharge summary indoor case papers (if any) and prescriptions issued by pe treating Medical practitioner or Network Provider\n",
+      "Optional Co-payment\n",
+      "Adventure Sports Cover\n",
+      "Home to Home Cover\n",
+      "Passport and Visa copy wip Entry Stamp of Country of Visit and exit Stamp from India\n",
+      "Extension to in-patient care\n",
+      "Ambulance Charge\n",
+      "FIR report of police (if applicable)\n",
+      "\n",
+      "## Out-patient treatment\n",
+      "\n",
+      "Cancer Screening & Mammographic Examination\n",
+      "Original bills and receipts for:\n",
+      "1. Charges paid towards Hospital accommodation, nursing facilities, and oper medical services rendered\n",
+      "2. Fees paid to pe Medical Practitioner and for special nursing charges\n",
+      "3. Charges incurred towards any and all test and / or examinations rendered in connection wip pe treatment\n",
+      "4. Charges incurred towards medicines or drugs purchased from a registered pharmacy oper pan pe Network provider duly supported by pe prescriptions of pe Medical Practitioner attending to pe Insured Person\n",
+      "5. Any oper document as required by pe Company to assist pe Claim\n",
+      "\n",
+      "## Medical evacuation\n",
+      "\n",
+      "Medical reports and transportation details issued by the evacuation agency, prescriptions and medical report by the attending Medical Practitioner furnishing the name of the Insured Person and details of treatment rendered along with the statement confirming the necessity of evacuation.\n",
+      "\n",
+      "Documentary proof for expenses incurred towards the Medical Evacuation.\n",
+      "\n",
+      "## Compassionate visit\n",
+      "\n",
+      "A certificate from the Medical Practitioner recommending the presence in the form of special assistance to be rendered by an additional member during the entire period of hospitalization. The certificate shall also specify the minimum period in which person is admitted in the hospital.\n",
+      "\n",
+      "Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
+      "\n",
+      "Stamped boarding pass with invoice used for the travel by the Immediate Family Member.\n",
+      "\n",
+      "Copy passport of Immediate Family Member with entry and exit stamp.\n",
+      "\n",
+      "## Escort of Minor Child\n",
+      "\n",
+      "A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
+      "\n",
+      "Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
+      "\n",
+      "Stamped Boarding pass used for the return travel of the child to the Country of Residence.\n",
+      "\n",
+      "Stamped Boarding pass of the attendant from the Country of Residence to the place of hospitalization (if attendant is necessary).\n",
+      "\n",
+      "Copy of passport of the child with entry and exit stamp.\n",
+      "\n",
+      "## Upgradation to Business Class\n",
+      "\n",
+      "A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
+      "\n",
+      "Discharge summary of the Hospital furnishing the details including the date of admission and date of discharge.\n",
+      "\n",
+      "Product Name: Travel infinity | Product UIN: NBHTGBP22011V012223\n",
+      "\n",
+      "\n",
+      "=========================================================\n",
+      "\n",
+      "\n",
+      "# Insurance Policy\n",
+      "\n",
+      "## Benefits:\n",
+      "\n",
+      "- For Inpatient treatment and in this condition when admitted to a hospital, the coverage is reimbursement for medical expenses incurred.\n",
+      "- For Hospital Daily Cash and in this condition when hospitalized, the coverage is daily cash benefit.\n",
+      "- For Waiver of Deductible and in this condition when a deductible is applicable, the coverage is waiver of the deductible amount.\n",
+      "- For Optional Co-payment and in this condition when a co-payment is required, the coverage is optional co-payment.\n",
+      "- For Adventure Sports Cover and in this condition when participating in adventure sports, the coverage is coverage for injuries related to adventure sports.\n",
+      "- For Home to Home Cover and in this condition when requiring medical evacuation, the coverage is assistance for repatriation to home country.\n",
+      "- For Extension to in-patient care and in this condition when extended hospital stay is necessary, the coverage is extension of coverage for in-patient care.\n",
+      "- For Ambulance Charge and in this condition when ambulance services are utilized, the coverage is reimbursement for ambulance charges.\n",
+      "- For Out-patient treatment and in this condition when receiving outpatient medical care, the coverage is reimbursement for outpatient medical expenses.\n",
+      "- For Cancer Screening & Mammographic Examination and in this condition when undergoing cancer screening or mammographic examination, the coverage is coverage for these preventive services.\n",
+      "- For New Born baby Cover and in this condition when a newborn is covered under the policy, the coverage is medical expenses coverage for the newborn.\n",
+      "- For Maternity and in this condition when maternity services are required, the coverage is coverage for maternity expenses.\n",
+      "- For Complete pre-existing disease cover and in this condition when seeking treatment for pre-existing conditions, the coverage is coverage for pre-existing conditions.\n",
+      "- For Medical sum insured replenishment in case of hospitalization due to accident and in this condition when hospitalized due to an accident, the coverage is replenishment of the sum insured.\n",
+      "- For Waiver of sublimit for insured above 60 years of age and in this condition when the insured is above 60 years of age, the coverage is waiver of sublimits.\n",
+      "- For Psychiatric Counseling and in this condition when seeking psychiatric counseling, the coverage is coverage for psychiatric counseling services.\n",
+      "- For Physiotherapy and in this condition when undergoing physiotherapy, the coverage is coverage for physiotherapy sessions.\n",
+      "- For Terrorism cover and in this condition when affected by terrorism, the coverage is coverage for medical expenses related to terrorism incidents.\n",
+      "- For Medical tele-consultation and in this condition when consulting a medical practitioner remotely, the coverage is coverage for tele-consultation services.\n",
+      "- For Medical evacuation and in this condition when requiring medical evacuation, the coverage is coverage for medical evacuation services.\n",
+      "- For Compassionate visit and in this condition when requiring a compassionate visit, the coverage is coverage for travel expenses for a family member to visit.\n",
+      "- For Escort of Minor Child and in this condition when escorting a minor child for medical treatment, the coverage is coverage for escort services for the child.\n",
+      "- For Upgradation to Business Class and in this condition when requiring upgradation to business class for medical travel, the coverage is coverage for upgradation to business class.\n"
+     ]
+    }
+   ],
+   "source": [
+    "target_page = 45\n",
+    "pages_vanilla = documents[0].text.split(\"\\n---\\n\")\n",
+    "pages_with_instructions = documents_with_instruction[0].text.split(\"\\n---\\n\")\n",
+    "\n",
+    "print(pages_vanilla[target_page])\n",
+    "print(\"\\n\\n=========================================================\\n\\n\")\n",
+    "print(pages_with_instructions[target_page])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "node_parser_instruction = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")\n",
+    "nodes_instruction = node_parser.get_nodes_from_documents(documents_with_instruction)\n",
+    "(\n",
+    "    base_nodes_instruction,\n",
+    "    objects_instruction,\n",
+    ") = node_parser_instruction.get_nodes_and_objects(nodes_instruction)\n",
+    "\n",
+    "recursive_index_instruction = VectorStoreIndex(\n",
+    "    nodes=base_nodes_instruction + objects_instruction\n",
+    ")\n",
+    "query_engine_instruction = recursive_index_instruction.as_query_engine(\n",
+    "    similarity_top_k=25\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Comparing Instruction-Augmented Parsing vs. Vanilla Parsing\n",
+    "\n",
+    "When we parse the document with natural language instructions to add context on insurance coverage, we are able to correctly answer a wide range of queries in our RAG pipeline. In contrast, a RAG pipeline built with the vanilla method is not able to answer these queries."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Vanilla:\n",
+      "You are covered for the amount you paid due to the trip delay, up to the limit specified in the certificate of insurance.\n",
+      "With instructions:\n",
+      "For Trip Delay coverage, you are covered for a fixed benefit amount as mentioned in the certificate of insurance for every block of hours of delay.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_1 = \"My trip was delayed and I paid 45, how much am I covered for?\"\n",
+    "\n",
+    "response_1 = query_engine.query(query_1)\n",
+    "print(\"Vanilla:\")\n",
+    "print(response_1)\n",
+    "\n",
+    "print(\"With instructions:\")\n",
+    "response_1_i = query_engine_instruction.query(query_1)\n",
+    "print(response_1_i)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Looking at the policy it says in list I that one expense not covered is Baby food"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Vanilla:\n",
+      "Baby food is not explicitly mentioned in the provided context information regarding insurance coverages and benefits.\n",
+      "With instructions:\n",
+      "Baby food is excluded from coverage according to the policy terms.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_2 = \"I just had a baby, is baby food covered?\"\n",
+    "\n",
+    "response_2 = query_engine.query(query_2)\n",
+    "print(\"Vanilla:\")\n",
+    "print(response_2)\n",
+    "\n",
+    "print(\"With instructions:\")\n",
+    "response_2_i = query_engine_instruction.query(query_2)\n",
+    "print(response_2_i)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Vanilla:\n",
+      "Gauze used in your operation would typically be covered under the \"Emergency In-patient Medical Treatment\" or \"Emergency In-patient Medical Treatment with OPD\" benefits of the policy.\n",
+      "With instructions:\n",
+      "Gauze is not covered for use in your operation as it falls under the category of items that are excluded from coverage in the insurance policy.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_3 = \"How is gauze used in my operation covered?\"\n",
+    "\n",
+    "response_3 = query_engine.query(query_3)\n",
+    "print(\"Vanilla:\")\n",
+    "print(response_3)\n",
+    "\n",
+    "print(\"With instructions:\")\n",
+    "response_3_i = query_engine_instruction.query(query_3)\n",
+    "print(response_3_i)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "llama_parse",
+   "language": "python",
+   "name": "llama_parse"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
@@ -40,18 +40,18 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "id": "0879301c-ff91-4431-941a-6c0ef7cd8fe2",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -63,9 +63,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "391e2d95-5569-4d73-9f16-5b59d7326f8d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.anthropic import Anthropic\n",
@@ -75,11 +73,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "id": "700f48e8-8b52-41f3-90f9-144d5fdd5c52",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import Settings\n",
@@ -102,12 +98,10 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "c39d408f-e885-4940-85c7-b09ca3bc7cb7",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -122,11 +116,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "9c9cd670-8229-4ad6-99a9-845bd82b7ec1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -146,11 +138,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "b26d21d1-05b5-4f49-b937-c13106a84015",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.schema import TextNode\n",
@@ -160,23 +150,16 @@
    "def get_text_nodes(json_list: List[dict]):\n",
    "    text_nodes = []\n",
    "    for idx, page in enumerate(json_list):\n",
-    "        text_node = TextNode(\n",
-    "            text=page[\"text\"],\n",
-    "            metadata={\n",
-    "                \"page\": page[\"page\"]\n",
-    "            }\n",
-    "        )\n",
+    "        text_node = TextNode(text=page[\"text\"], metadata={\"page\": page[\"page\"]})\n",
    "        text_nodes.append(text_node)\n",
    "    return text_nodes"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "id": "364a3276-d2db-4aee-9bc6-617ffd726d25",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "text_nodes = get_text_nodes(json_list)"
@@ -194,7 +177,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "36012145-5521-4ddb-a53e-df9ebd1ca8dd",
   "metadata": {},
   "outputs": [
@@ -223,13 +206,10 @@
    "    for image_dict in image_dicts:\n",
    "        image_doc = ImageDocument(image_path=image_dict[\"path\"])\n",
    "        response = anthropic_mm_llm.complete(\n",
-    "            prompt=\"Describe the images as an alternative text\",\n",
+    "            prompt=\"Describe the images as alt text\",\n",
    "            image_documents=[image_doc],\n",
    "        )\n",
-    "        text_node = TextNode(\n",
-    "            text=str(response),\n",
-    "            metadata={\"path\": image_dict[\"path\"]}\n",
-    "        )\n",
+    "        text_node = TextNode(text=str(response), metadata={\"path\": image_dict[\"path\"]})\n",
    "        img_text_nodes.append(text_node)\n",
    "    return img_text_nodes"
   ]
@@ -238,9 +218,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "38f25045-6102-4920-9cd0-42b0ae6c872f",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "image_text_nodes = get_image_text_nodes(json_objs)"
@@ -248,11 +226,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 60,
+   "execution_count": null,
   "id": "4683c97a-da06-408a-9fe9-7e3c0aceb77d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "data": {
@@ -260,7 +236,7 @@
       "'The image shows a bar graph titled \"Monthly Active Platform Consumers (in millions)\". The graph displays data from Q2 2020 to Q1 2022 over 8 quarters. The number of monthly active platform consumers starts at 55 million in Q2 2020 and steadily increases each quarter, reaching 115 million by Q1 2022. The graph illustrates consistent quarter-over-quarter growth in this metric over the nearly 2 year time period shown.'"
      ]
     },
-     "execution_count": 60,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -281,7 +257,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 68,
+   "execution_count": null,
   "id": "939aec6c-064a-4319-b2dc-70cc4a304c06",
   "metadata": {},
   "outputs": [],
@@ -293,7 +269,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 69,
+   "execution_count": null,
   "id": "529340d5-9319-4cdf-8ee1-bbd01ed00226",
   "metadata": {},
   "outputs": [],
@@ -303,11 +279,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 71,
+   "execution_count": null,
   "id": "81d7ff30-5a87-44da-880d-4b1f41434d90",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -320,18 +294,18 @@
    }
   ],
   "source": [
-    "# ask question over image! \n",
-    "response = query_engine.query(\"What does the bar graph titled 'Monthly Active Platform Consumers' show?\") \n",
-    "print(str(response)) "
+    "# ask question over image!\n",
+    "response = query_engine.query(\n",
+    "    \"What does the bar graph titled 'Monthly Active Platform Consumers' show?\"\n",
+    ")\n",
+    "print(str(response))"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 72,
+   "execution_count": null,
   "id": "c4f14ad8-6bfd-49d9-b3d5-7215cf0e4ac1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -360,25 +334,17 @@
    }
   ],
   "source": [
-    "# ask question over text! \n",
-    "response = query_engine.query(\"What are the main risk factors for Uber?\") \n",
-    "print(str(response)) "
+    "# ask question over text!\n",
+    "response = query_engine.query(\"What are the main risk factors for Uber?\")\n",
+    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "82ea880b-a0c7-410c-94c7-8fb3ac96c30c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
-   "display_name": "llama_parse",
+   "display_name": "llama-parse-aNC435Vv-py3.10",
   "language": "python",
-   "name": "llama_parse"
+   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
@@ -389,8 +355,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -21,16 +21,14 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "87322210-c21c-43d6-b459-2e8a828ac576",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
@@ -39,6 +37,7 @@
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
   ]
  },
@@ -56,9 +55,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "e81e0a08-3a99-42e6-adcc-00bb4ce1c3d4",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "!wget \"https://www.dropbox.com/scl/fi/fxg17log5ydwoflhxmgrb/treasury_report.pdf?rlkey=mdintk0o2uuzkple26vc4v6fd&dl=1\" -O treasury_report.pdf"
@@ -66,11 +63,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "ecfc578c-3c7f-4ec1-aa06-51565c28632b",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -84,20 +79,15 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"fr\"\n",
-    ")\n",
+    "parser = LlamaParse(result_type=\"text\", language=\"fr\")\n",
    "documents = parser.load_data(\"./treasury_report.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "0c37db27-3496-4a59-918b-701c9ad7706d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -209,11 +199,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
   "id": "ac332ea3-cfff-4216-b292-62410a26c336",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -248,11 +236,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
   "id": "45235b17-08f0-48f1-92aa-06711225860b",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -266,20 +252,15 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"ch_sim\"\n",
-    ")\n",
+    "parser = LlamaParse(result_type=\"text\", language=\"ch_sim\")\n",
    "documents = parser.load_data(\"./chinese_pdf.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
   "id": "f0d546cc-6549-4cf5-8b37-0896f4e8d43d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -401,9 +382,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "640f0679-7f7e-4b0a-a46d-b099ae382fe2",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# download another copy with a different name to avoid hitting pdf cache\n",
@@ -412,11 +391,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": null,
   "id": "bfcacf90-ca67-4bfd-b023-be0af2cb18c5",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -429,10 +406,7 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "base_parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"en\"\n",
-    ")\n",
+    "base_parser = LlamaParse(result_type=\"text\", language=\"en\")\n",
    "base_documents = parser.load_data(\"./chinese_pdf2.pdf\")"
   ]
  },
@@ -440,21 +414,11 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "b264ed4e-647a-4f51-9f79-fdf82b76762a",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "print(base_documents[0].get_content()[1000:10000])"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d9f02762-bb97-4e0e-8268-ccc00612a974",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -472,8 +436,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -0,0 +1,368 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LlamaParse With MongoDB\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_mongodb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "In this notebook, we provide a straightforward example of using LlamaParse with MongoDB Atlas VectorSearch.\n",
+    "\n",
+    "We illustrate the process of using llama-parse to parse a PDF document, then index the document with a MongoDB vector store, and subsequently perform basic queries against this store.\n",
+    "\n",
+    "This notebook is structured similarly to quick start guides, aiming to introduce users to utilizing llama-parse in conjunction with a MongoDB Atlas VectorSearch."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install llama-index llama-parse\n",
+    "%pip install llama-index-vector-stores-mongodb llama-index-llms-openai"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Setup API Keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\n",
+    "    \"LLAMA_CLOUD_API_KEY\"\n",
+    "] = \"\"  # Get it from https://cloud.llamaindex.ai/api-key\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"\"  # Get it from https://platform.openai.com/api-keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "import requests\n",
+    "import pymongo\n",
+    "\n",
+    "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
+    "from llama_parse import LlamaParse\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.core import VectorStoreIndex, StorageContext\n",
+    "from llama_index.core.node_parser import SimpleNodeParser"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Download Document\n",
+    "\n",
+    "We will use `Attention is all you need` paper."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Download complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# The URL of the file you want to download\n",
+    "url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
+    "# The local path where you want to save the file\n",
+    "file_path = \"./attention.pdf\"\n",
+    "\n",
+    "# Perform the HTTP request\n",
+    "response = requests.get(url)\n",
+    "\n",
+    "# Check if the request was successful\n",
+    "if response.status_code == 200:\n",
+    "    # Open the file in binary write mode and save the content\n",
+    "    with open(file_path, \"wb\") as file:\n",
+    "        file.write(response.content)\n",
+    "    print(\"Download complete.\")\n",
+    "else:\n",
+    "    print(\"Error downloading the file.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Parse the document using `LlamaParse`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 09a49745-9f21-4190-9de8-27e4e1a4bdf5\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = LlamaParse(result_type=\"text\").load_data(file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "rmer - model architecture.\n",
+      "The Transformer follows this overall architecture using stacked self-attention and point-wise, fully\n",
+      "connected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\n",
+      "respectively.\n",
+      "3.1   Encoder and Decoder Stacks\n",
+      "Encoder:     The encoder is composed of a stack of N = 6 identical layers. Each layer has two\n",
+      "sub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\n",
+      "wise fully connected feed-forward network. We employ a residual connection [11] around each of\n",
+      "the two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\n",
+      "LayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\n",
+      "itself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\n",
+      "layers, produce outputs of dimension dmodel = 512.\n",
+      "Decoder:    The decoder is also composed of a stack of N = 6 identical layers. In addition \n"
+     ]
+    }
+   ],
+   "source": [
+    "# Take a quick look at some of the parsed text from the document:\n",
+    "print(documents[0].get_content()[10000:11000])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create `MongoDBAtlasVectorSearch`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mongo_uri = os.environ[\"MONGO_URI\"]\n",
+    "\n",
+    "mongodb_client = pymongo.MongoClient(mongo_uri)\n",
+    "mongodb_vector_store = MongoDBAtlasVectorSearch(mongodb_client)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create nodes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "node_parser = SimpleNodeParser()\n",
+    "\n",
+    "nodes = node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create Index and Query Engine."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(vector_store=mongodb_vector_store)\n",
+    "\n",
+    "index = VectorStoreIndex(\n",
+    "    nodes=nodes,\n",
+    "    storage_context=storage_context,\n",
+    "    embed_model=OpenAIEmbedding(),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_engine = index.as_query_engine(similarity_top_k=2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Test Query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "***********New LlamaParse+ Basic Query Engine***********\n",
+      "The BLEU score on the WMT 2014 English-to-German translation task is 28.4.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query = \"What is BLEU score on the WMT 2014 English-to-German translation task?\"\n",
+    "\n",
+    "response = query_engine.query(query)\n",
+    "print(\"\\n***********New LlamaParse+ Basic Query Engine***********\")\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "We varied the learning\n",
+      "rate over the course of training, according to the formula:\n",
+      "               lrate = d−0.5                                                                          (3)\n",
+      "                         model · min(step_num−0.5, step_num · warmup_steps−1.5)\n",
+      "This corresponds to increasing the learning rate linearly for the first warmup_steps training steps,\n",
+      "and decreasing it thereafter proportionally to the inverse square root of the step number. We used\n",
+      "warmup_steps = 4000.\n",
+      "5.4   Regularization\n",
+      "We employ three types of regularization during training:\n",
+      "                                                    7\n",
+      "---\n",
+      "Table 2: The Transformer achieves better BLEU scores than previous state-of-the-art models on the\n",
+      "English-to-German and English-to-French newstest2014 tests at a fraction of the training cost.\n",
+      "        Model                                          BLEU               Training Cost (FLOPs)\n",
+      "                                                 EN-DE      EN-FR          EN-DE         EN-FR\n",
+      "        ByteNet [18]                              23.75\n",
+      "        Deep-Att + PosUnk [39]                               39.2                       1.0 · 1020\n",
+      "        GNMT + RL [38]                            24.6       39.92        2.3 · 1019    1.4 · 1020\n",
+      "        ConvS2S [9]                               25.16      40.46        9.6 · 1018    1.5 · 1020\n",
+      "        MoE [32]                                  26.03      40.56        2.0 · 1019    1.2 · 1020\n",
+      "        Deep-Att + PosUnk Ensemble [39]                      40.4                       8.0 · 1020\n",
+      "        GNMT + RL Ensemble [38]                   26.30      41.16        1.8 · 1020    1.1 · 1021\n",
+      "        ConvS2S Ensemble [9]                      26.36      41.29        7.7 · 1019    1.2 · 1021\n",
+      "        Transformer (base model)                  27.3       38.1               3.3 · 1018\n",
+      "        Transformer (big)                         28.4       41.8                2.3 · 1019\n",
+      "Residual Dropout       We apply dropout [33] to the output of each sub-layer, before it is added to the\n",
+      "sub-layer input and normalized. In addition, we apply dropout to the sums of the embeddings and the\n",
+      "positional encodings in both the encoder and decoder stacks. For the base model, we use a rate of\n",
+      "Pdrop = 0.1.\n",
+      "Label Smoothing        During training, we employed label smoothing of value ϵls = 0.1 [36]. This\n",
+      "hurts perplexity, as the model learns to be more unsure, but improves accuracy and BLEU score.\n",
+      "6    Results\n",
+      "6.1   Machine Translation\n",
+      "On the WMT 2014 English-to-German translation task, the big transformer model (Transformer (big)\n",
+      "in Table 2) outperforms the best previously reported models (including ensembles) by more than 2.0\n",
+      "BLEU, establishing a new state-of-the-art BLEU score of 28.4. The configuration of this model is\n",
+      "listed in the bottom line of Table 3. Training took 3.5 days on 8 P100 GPUs. Even our base model\n",
+      "surpasses all previously published models and ensembles, at a fraction of the training cost of any of\n",
+      "the competitive models.\n",
+      "On the WMT 2014 English-to-French translation task, our big model achieves a BLEU score of 41.0,\n",
+      "outperforming all of the previously published single models, at less than 1/4 the training cost of the\n",
+      "previous state-of-the-art model. The Transformer (big) model trained for English-to-French used\n",
+      "dropout rate Pdrop = 0.1, instead of 0.3.\n",
+      "For the base models, we used a single model obtained by averaging the last 5 checkpoints, which\n",
+      "were written at 10-minute intervals. For the big models, we averaged the last 20 checkpoints. We\n",
+      "used beam search with a beam size of 4 and length penalty α = 0.6 [38]. These hyperparameters\n",
+      "were chosen after experimentation on the development set. We set the maximum output length during\n",
+      "inference to input length + 50, but terminate early when possible [38].\n",
+      "Table 2 summarizes our results and compares our translation quality and training costs to other model\n",
+      "architectures from the literature.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Take a look at one of the source nodes from the response\n",
+    "print(response.source_nodes[0].get_content())"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "anthropic_env",
+   "language": "python",
+   "name": "anthropic_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
@@ -0,0 +1,544 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LlamaParse - Parsing comic books with parsing intructions\n",
+    "Parsing intructions allow you to instruct our parsing model the same way you would instruct an LLM!\n",
+    "\n",
+    "They can be useful to help the parser get better results on complex document layouts, to extract data in a specific format, or to transform the document in other ways.\n",
+    "\n",
+    "Using Parsing Instruction you will get better results out of LlamaParse on complicated documents, and also be able to simplify your application code."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Installation\n",
+    "\n",
+    "Parsing instructions are part of the llamaParse API. They can be accessed by directly specifying the parsing_instruction parameter in the API or by using the LlamaParse python module (which we will use for this tutorial).\n",
+    "\n",
+    "To install llama-parse, just get it from PIP:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting llama-parse\n",
+      "  Downloading llama_parse-0.3.8-py3-none-any.whl (6.7 kB)\n",
+      "Collecting llama-index-core>=0.10.7 (from llama-parse)\n",
+      "  Downloading llama_index_core-0.10.19-py3-none-any.whl (15.3 MB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m31.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (6.0.1)\n",
+      "Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2.0.28)\n",
+      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.9.3)\n",
+      "Collecting dataclasses-json (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n",
+      "Collecting deprecated>=1.2.9.3 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
+      "Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2023.6.0)\n",
+      "Collecting httpx (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hCollecting llamaindex-py-client<0.2.0,>=0.1.13 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading llamaindex_py_client-0.1.13-py3-none-any.whl (107 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.0/108.0 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.6.0)\n",
+      "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.2.1)\n",
+      "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (3.8.1)\n",
+      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.25.2)\n",
+      "Collecting openai>=1.1.0 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading openai-1.13.3-py3-none-any.whl (227 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.4/227.4 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (1.5.3)\n",
+      "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (9.4.0)\n",
+      "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (2.31.0)\n",
+      "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (8.2.3)\n",
+      "Collecting tiktoken>=0.3.3 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (4.66.2)\n",
+      "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core>=0.10.7->llama-parse) (4.10.0)\n",
+      "Collecting typing-inspect>=0.8.0 (from llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.3.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (23.2.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.4.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (6.0.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (1.9.4)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core>=0.10.7->llama-parse) (4.0.3)\n",
+      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.9.3->llama-index-core>=0.10.7->llama-parse) (1.14.1)\n",
+      "Requirement already satisfied: pydantic>=1.10 in /usr/local/lib/python3.10/dist-packages (from llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (2.6.3)\n",
+      "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (3.7.1)\n",
+      "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (2024.2.2)\n",
+      "Collecting httpcore==1.* (from httpx->llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (3.6)\n",
+      "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core>=0.10.7->llama-parse) (1.3.1)\n",
+      "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx->llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (8.1.7)\n",
+      "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (1.3.2)\n",
+      "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core>=0.10.7->llama-parse) (2023.12.25)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core>=0.10.7->llama-parse) (1.7.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core>=0.10.7->llama-parse) (3.3.2)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core>=0.10.7->llama-parse) (2.0.7)\n",
+      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core>=0.10.7->llama-parse) (3.0.3)\n",
+      "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
+      "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core>=0.10.7->llama-parse)\n",
+      "  Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n",
+      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core>=0.10.7->llama-parse) (2.8.2)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core>=0.10.7->llama-parse) (2023.4)\n",
+      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core>=0.10.7->llama-parse) (1.2.0)\n",
+      "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core>=0.10.7->llama-parse) (23.2)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (0.6.0)\n",
+      "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llamaindex-py-client<0.2.0,>=0.1.13->llama-index-core>=0.10.7->llama-parse) (2.16.3)\n",
+      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->llama-index-core>=0.10.7->llama-parse) (1.16.0)\n",
+      "Installing collected packages: dirtyjson, mypy-extensions, marshmallow, h11, deprecated, typing-inspect, tiktoken, httpcore, httpx, dataclasses-json, openai, llamaindex-py-client, llama-index-core, llama-parse\n",
+      "Successfully installed dataclasses-json-0.6.4 deprecated-1.2.14 dirtyjson-1.0.8 h11-0.14.0 httpcore-1.0.4 httpx-0.27.0 llama-index-core-0.10.19 llama-parse-0.3.8 llamaindex-py-client-0.1.13 marshmallow-3.21.1 mypy-extensions-1.0.0 openai-1.13.3 tiktoken-0.6.0 typing-inspect-0.9.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "%pip install llama-parse"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## API key\n",
+    "\n",
+    "The use of LlamaParse requires an API key which you can get here: https://cloud.llamaindex.ai/parse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Async (Notebook only)\n",
+    "llama-parse is async-first, so running the code in a notebook requires the use of nest_asyncio\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Import the package"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_parse import LlamaParse"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Using llamaparse for getting better results (on Manga!)\n",
+    "\n",
+    "Sometimes the layout of a page is unusual and you will get sub-optimal reading order results with LlamaParse. For example, when parsing manga you expect the reading order to be right to left even if the content is in English!"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's download an extract of a great manga \"The manga guide to calculus\", by Hiroyuki Kojima (https://www.amazon.com/Manga-Guide-Calculus-Hiroyuki-Kojima/dp/1593271948)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2024-03-13 13:57:19--  https://drive.usercontent.google.com/uc?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\n",
+      "Resolving drive.usercontent.google.com (drive.usercontent.google.com)... 173.194.211.132, 2607:f8b0:400c:c10::84\n",
+      "Connecting to drive.usercontent.google.com (drive.usercontent.google.com)|173.194.211.132|:443... connected.\n",
+      "HTTP request sent, awaiting response... 303 See Other\n",
+      "Location: https://drive.usercontent.google.com/download?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download [following]\n",
+      "--2024-03-13 13:57:19--  https://drive.usercontent.google.com/download?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\n",
+      "Reusing existing connection to drive.usercontent.google.com:443.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 3041634 (2.9M) [application/octet-stream]\n",
+      "Saving to: ‘./manga.pdf’\n",
+      "\n",
+      "./manga.pdf         100%[===================>]   2.90M  --.-KB/s    in 0.04s   \n",
+      "\n",
+      "2024-03-13 13:57:20 (78.6 MB/s) - ‘./manga.pdf’ saved [3041634/3041634]\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "! wget \"https://drive.usercontent.google.com/uc?id=1tZJhcpepLRdQFJFCFX50QIqLyLgqzZsY&export=download\" -O ./manga.pdf"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Without parsing instructions\n",
+    "For the sake of comparison, let's first parse without any instructions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 25bf4202-78d8-4705-88cf-c616ae7c82af\n"
+     ]
+    }
+   ],
+   "source": [
+    "vanilaParsing = LlamaParse(result_type=\"markdown\").load_data(\"./manga.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "As you can see below, LlamaParse is not doing a great job here. It is interpreting the grid of comic panels as a table, and trying to fit the dialogue into a table. It's very hard to follow."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "The Asagake Times Sanda-Cho Distributor\n",
+      "\n",
+      "A newspaper distributor? do I have the wrong map?\n",
+      "\n",
+      "You’re looking It’s next for the Sanda-cho door. branch office? Everybody mistakes us for the office because we are larger. What Is a Function? 3\n",
+      "---\n",
+      "## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
+      "\n",
+      "|1.|Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is|\n",
+      "|---|---|\n",
+      "| |lim ε→0 (f(a + ε) - f(a)) / ε = lim ε→0 (α - α) = lim ε→0 0 = 0|\n",
+      "| |Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.|\n",
+      "\n",
+      "Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
+      "\n",
+      "|2.|Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is|\n",
+      "|---|---|\n",
+      "| |lim ε→0 (f(α + ε) - f(a)) = \n"
+     ]
+    }
+   ],
+   "source": [
+    "print(vanilaParsing[0].text[100:1000])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Using parsing instructions\n",
+    "Let's try to parse the manga with custom instructions:\n",
+    "\n",
+    "\"The provided document is a manga comic book. Most pages do NOT have a title. It does not contain tables. Try to reconstruct the dialogue spoken in a cohesive way.\"\n",
+    "\n",
+    "To do so just pass the parsing instruction as a parameter to LlamaParse:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 88ab273e-b2a7-4f84-8e72-e9367cf6b114\n",
+      "."
+     ]
+    }
+   ],
+   "source": [
+    "parsingInstructionManga = \"\"\"The provided document is a manga comic book. Most pages do NOT have a title.\n",
+    "It does not contain tables.\n",
+    "Try to reconstruct the dialogue spoken in a cohesive way.\"\"\"\n",
+    "withInstructionParsing = LlamaParse(\n",
+    "    result_type=\"markdown\", parsing_instruction=parsingInstructionManga\n",
+    ").load_data(\"./manga.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's see how it compare with page 3! We encourage you to play with the target page and explore other pages. As you will see, the parsing instruction allowed LlamaParse to make sense of the document!\n",
+    "\n",
+    "<img src=\"https://drive.usercontent.google.com/download?id=1M87rXTIZE8d5v7aHmVZVW6gW3eDGq6ks&authuser=0\" />\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The Asagake Times Sanda-Cho Distributor\n",
+      "\n",
+      "A newspaper distributor? do I have the wrong map?\n",
+      "\n",
+      "You’re looking It’s next for the Sanda-cho door. branch office? Everybody mistakes us for the office because we are larger. What Is a Function? 3\n",
+      "\n",
+      "\n",
+      "------------------------------------------------------------\n",
+      "\n",
+      "\n",
+      "# The Asagake Times\n",
+      "\n",
+      "Sanda-Cho Distributor\n",
+      "\n",
+      "A newspaper distributor?\n",
+      "\n",
+      "Do I have the wrong map?\n",
+      "\n",
+      "You're looking for the Sanda-cho branch office?\n",
+      "\n",
+      "It's next door.\n",
+      "\n",
+      "Everybody mistakes us for the office because we are larger.\n",
+      "\n",
+      "What Is a Function? 3\n"
+     ]
+    }
+   ],
+   "source": [
+    "target_page = 1\n",
+    "print(vanilaParsing[0].text.split(\"\\n---\\n\")[target_page])\n",
+    "print(\"\\n\\n------------------------------------------------------------\\n\\n\")\n",
+    "print(withInstructionParsing[0].text.split(\"\\n---\\n\")[target_page])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Math - doing more with parsing instuction!\n",
+    "\n",
+    "But this manga is about math and full of equations, why not ask the parser to output them in **LaTeX**?\n",
+    "\n",
+    "<img src=\"https://drive.usercontent.google.com/download?id=1tze3xcQ7axVA-vC_iZeAj_GvYcyNuYDa&authuser=0\" />"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 3a055e64-d91e-484e-b9b0-99a2e637c08d\n",
+      "."
+     ]
+    }
+   ],
+   "source": [
+    "parsingInstructionMangaLatex = \"\"\"The provided document is a manga comic book. Most pages do NOT have a title.\n",
+    "It does not contain tables.\n",
+    "Try to reconstruct the dialogue spoken in a cohesive way.\n",
+    "Output any math equation in LATEX markdown (between $$)\"\"\"\n",
+    "withLatex = LlamaParse(\n",
+    "    result_type=\"markdown\", parsing_instruction=parsingInstructionMangaLatex\n",
+    ").load_data(\"./manga.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "[Without instruction]------------------------------------------------------------\n",
+      "\n",
+      "\n",
+      "## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
+      "\n",
+      "|1.|Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is|\n",
+      "|---|---|\n",
+      "| |lim ε→0 (f(a + ε) - f(a)) / ε = lim ε→0 (α - α) = lim ε→0 0 = 0|\n",
+      "| |Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.|\n",
+      "\n",
+      "Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
+      "\n",
+      "|2.|Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is|\n",
+      "|---|---|\n",
+      "| |lim ε→0 (f(α + ε) - f(a)) = lim ε→0 (α(a + ε) + β - (αa + β)) = lim ε→0 α = α|\n",
+      "| |Thus, the derivative of f(x) is f′(x) = α, a constant value. This result should also be intuitive—linear functions have a constant rate of change by definition.|\n",
+      "\n",
+      "|3.|Let’s find the derivative of f(x) = x^2, which appeared in the story. The differential coefficient of f(x) at x = a is|\n",
+      "|---|---|\n",
+      "| |lim ε→0 ((a + ε)^2 - a^2) / ε = lim (a^2 + 2aε + ε^2 - a^2) / ε = lim (2aε + ε^2) = lim (2a + ε) = 2a|\n",
+      "| |Thus, the differential coefficient of f(x) at x = a is 2a, or f′(a) = 2a. Therefore, the derivative of f(x) is f′(x) = 2x.|\n",
+      "\n",
+      "## Summary\n",
+      "\n",
+      "- The calculation of a limit that appears in calculus is simply a formula calculating an error.\n",
+      "- A limit is used to obtain a derivative.\n",
+      "- The derivative is the slope of the tangent line at a given point.\n",
+      "- The derivative is nothing but the rate of change.\n",
+      "\n",
+      "## Chapter 1 Let’s Differentiate a Function!\n",
+      "\n",
+      "\n",
+      "[With instruction to output math in LATEX!]------------------------------------------------------------\n",
+      "\n",
+      "\n",
+      "# Derivative of Constant, Linear, or Quadratic Function\n",
+      "\n",
+      "## Calculating the Derivative of a Constant, Linear, or Quadratic Function\n",
+      "\n",
+      "1. Let’s find the derivative of constant function f(x) = α. The differential coefficient of f(x) at x = a is\n",
+      "\n",
+      "$$\n",
+      "\\begin{align*}\n",
+      "&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(a + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\frac{\\alpha - \\alpha}{\\varepsilon} = \\lim_{{\\varepsilon \\to 0}} 0 = 0 \\\\\n",
+      "\\end{align*}\n",
+      "$$\n",
+      "Thus, the derivative of f(x) is f′(x) = 0. This makes sense, since our function is constant—the rate of change is 0.\n",
+      "\n",
+      "Note: The differential coefficient of f(x) at x = a is often simply called the derivative of f(x) at x = a, or just f′(a).\n",
+      "\n",
+      "2. Let’s calculate the derivative of linear function f(x) = αx + β. The derivative of f(x) at x = α is\n",
+      "\n",
+      "$$\n",
+      "\\begin{align*}\n",
+      "&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(\\alpha + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\frac{\\alpha(a + \\varepsilon) + \\beta - (\\alpha a + \\beta)}{\\varepsilon} = \\lim_{{\\varepsilon \\to 0}} \\alpha = \\alpha \\\\\n",
+      "\\end{align*}\n",
+      "$$\n",
+      "Thus, the derivative of f(x) is f′(x) = α, a constant value. This result should also be intuitive—linear functions have a constant rate of change by definition.\n",
+      "\n",
+      "3. Let’s find the derivative of f(x) = x2. The differential coefficient of f(x) at x = a is\n",
+      "\n",
+      "$$\n",
+      "\\begin{align*}\n",
+      "&\\lim_{{\\varepsilon \\to 0}} \\left( \\frac{f(a + \\varepsilon) - f(a)}{\\varepsilon} \\right) = \\lim_{{\\varepsilon \\to 0}} \\left( (a + \\varepsilon)^2 - a^2 \\right) = \\lim_{{\\varepsilon \\to 0}} 2a\\varepsilon + \\varepsilon = \\lim_{{\\varepsilon \\to 0}} (2a + \\varepsilon) = 2a \\\\\n",
+      "\\end{align*}\n",
+      "$$\n",
+      "Thus, the differential coefficient of f(x) at x = a is 2a, or f′(a) = 2a. Therefore, the derivative of f(x) is f′(x) = 2x.\n",
+      "\n",
+      "### Summary\n",
+      "\n",
+      "- The calculation of a limit that appears in calculus is simply a formula calculating an error.\n",
+      "- A limit is used to obtain a derivative.\n",
+      "- The derivative is the slope of the tangent line at a given point.\n",
+      "- The derivative is nothing but the rate of change.\n"
+     ]
+    }
+   ],
+   "source": [
+    "target_page = 2\n",
+    "print(\n",
+    "    \"\\n\\n[Without instruction]------------------------------------------------------------\\n\\n\"\n",
+    ")\n",
+    "print(vanilaParsing[0].text.split(\"\\n---\\n\")[target_page])\n",
+    "print(\n",
+    "    \"\\n\\n[With instruction to output math in LATEX!]------------------------------------------------------------\\n\\n\"\n",
+    ")\n",
+    "print(withLatex[0].text.split(\"\\n---\\n\")[target_page])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And here is the result as rendered by https://upmath.me/ .\n",
+    "\n",
+    "\n",
+    "<img src=\"https://drive.usercontent.google.com/download?id=1qGo5bMGYOiIC9MnprcgEByaYjU9YII2Q&authuser=0\" />\n",
+    "\n",
+    "\n",
+    "Over this short notebook we saw how to use parsing instructions to increase the quality and accuracy of parsing with LLamaParse!"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
@@ -28,13 +28,13 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-question-gen-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-question-gen-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
   ]
  },
  {
@@ -56,15 +56,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -74,7 +76,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -83,11 +85,11 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
-    "Settings.embed_model = embed_model\n"
+    "Settings.embed_model = embed_model"
   ]
  },
  {
@@ -109,8 +111,8 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "docs_2021 = LlamaParse(result_type=\"markdown\").load_data('./apple_2021_10k.pdf')\n",
-    "docs_2020 = LlamaParse(result_type=\"markdown\").load_data('./apple_2020_10k.pdf')"
+    "docs_2021 = LlamaParse(result_type=\"markdown\").load_data(\"./apple_2021_10k.pdf\")\n",
+    "docs_2020 = LlamaParse(result_type=\"markdown\").load_data(\"./apple_2020_10k.pdf\")"
   ]
  },
  {
@@ -127,31 +129,34 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
    "    model=\"BAAI/bge-reranker-large\",\n",
    ")\n",
    "\n",
+    "\n",
    "def create_query_engine_over_doc(docs, nodes_save_path=None):\n",
    "    \"\"\"Big function to go from document path -> recursive retriever.\"\"\"\n",
    "    if nodes_save_path is not None and os.path.exists(nodes_save_path):\n",
@@ -161,16 +166,13 @@
    "        if nodes_save_path is not None:\n",
    "            pickle.dump(raw_nodes, open(nodes_save_path, \"wb\"))\n",
    "\n",
-    "    base_nodes, objects = node_parser.get_nodes_and_objects(\n",
-    "        raw_nodes\n",
-    "    )\n",
+    "    base_nodes, objects = node_parser.get_nodes_and_objects(raw_nodes)\n",
    "\n",
    "    ### Construct Retrievers\n",
    "    # construct top-level vector index + query engine\n",
-    "    vector_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "    vector_index = VectorStoreIndex(nodes=base_nodes + objects)\n",
    "    query_engine = vector_index.as_query_engine(\n",
-    "        similarity_top_k=15,\n",
-    "        node_postprocessors=[reranker]\n",
+    "        similarity_top_k=15, node_postprocessors=[reranker]\n",
    "    )\n",
    "    return query_engine, base_nodes"
   ]
@@ -178,9 +180,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine_2021, nodes_2021 = create_query_engine_over_doc(\n",
@@ -193,10 +193,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 38,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
@@ -209,18 +207,14 @@
    "        query_engine=query_engine_2021,\n",
    "        metadata=ToolMetadata(\n",
    "            name=\"apple_2021_10k\",\n",
-    "            description=(\n",
-    "                \"Provides information about Apple financials for year 2021\"\n",
-    "            ),\n",
+    "            description=(\"Provides information about Apple financials for year 2021\"),\n",
    "        ),\n",
    "    ),\n",
    "    QueryEngineTool(\n",
    "        query_engine=query_engine_2020,\n",
    "        metadata=ToolMetadata(\n",
    "            name=\"apple_2020_10k\",\n",
-    "            description=(\n",
-    "                \"Provides information about Apple financials for year 2020\"\n",
-    "            ),\n",
+    "            description=(\"Provides information about Apple financials for year 2020\"),\n",
    "        ),\n",
    "    ),\n",
    "]\n",
@@ -241,10 +235,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 41,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -271,10 +263,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 40,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -290,10 +280,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 42,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -316,10 +304,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -342,10 +328,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 45,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -358,13 +342,6 @@
   "source": [
    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -382,8 +359,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -18,12 +18,11 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "14cdcfaf-88b4-4489-9910-e362e0ccec53",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "from llama_parse import LlamaParse"
@@ -31,12 +30,13 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "6f5b5841-dd3e-4169-9bd4-6a672b5b34ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\""
   ]
  },
@@ -68,7 +68,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "0dd0f860-8e92-43a7-9443-ad1a4fb9365c",
   "metadata": {},
   "outputs": [],
@@ -78,7 +78,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "fd932bef-ba82-4449-b7a0-5c2a9b55089f",
   "metadata": {},
   "outputs": [
@@ -104,7 +104,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "id": "2a73e553-2194-4ac9-9764-0edab0d6fdce",
   "metadata": {},
   "outputs": [
@@ -308,11 +308,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "c779547f-e4f7-4c84-9786-2b6b749827ab",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import VectorStoreIndex"
@@ -320,7 +318,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "id": "68b3a95e-ce19-4df1-9fdd-e6efb2fc423a",
   "metadata": {},
   "outputs": [],
@@ -330,11 +328,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "id": "a2ae28f6-4b3a-4130-8e65-0921b7678739",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = index.as_query_engine()"
@@ -342,23 +338,21 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "id": "232091ee-aa22-4f51-838c-410024acc344",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "response = query_engine.query(\"What are some response quality challenges with naive RAG?\") "
+    "response = query_engine.query(\n",
+    "    \"What are some response quality challenges with naive RAG?\"\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "id": "75f32aa7-c308-4221-af60-779822cfdba1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -371,14 +365,6 @@
   "source": [
    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d309d8fb-750a-4393-a1b2-67b14b7c121f",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -396,8 +382,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -2,9 +2,7 @@
 "cells": [
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "eld1dKaN7P8B"
-   },
+   "metadata": {},
   "source": [
    "# LlamaParse - Parsing Financial Powerpoints 📊\n",
    "\n",
@@ -13,9 +11,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "goB1sV8zu_Xl"
-   },
+   "metadata": {},
   "source": [
    "## Installation\n",
    "\n",
@@ -27,25 +23,17 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "7Y3_BwQLu-qK",
-    "outputId": "b1129c52-7a70-44cc-ad03-1f8d3a8c794a"
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-parse\n",
-    "!pip install torch transformers python-pptx Pillow"
+    "%pip install llama-index\n",
+    "%pip install llama-parse\n",
+    "%pip install torch transformers python-pptx Pillow"
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "i-Rg2D_Rvf2i"
-   },
+   "metadata": {},
   "source": [
    "## API Key\n",
    "\n",
@@ -55,32 +43,26 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "id": "af6i2P1vuU-U"
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "p8Eq-aX-wAEo"
-   },
+   "metadata": {},
   "source": [
    "**NOTE**: Since LlamaParse is natively async, running the sync code in a notebook requires the use of nest_asyncio.\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "id": "4OB0BkTqv_0l",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
@@ -90,9 +72,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "dz927ecMyYo_"
-   },
+   "metadata": {},
   "source": [
    "## Importing the package\n",
    "\n",
@@ -102,10 +82,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "id": "nSW-6sEwyXwx",
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_parse import LlamaParse"
@@ -113,9 +90,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "l_D4YsAHwUSk"
-   },
+   "metadata": {},
   "source": [
    "## Using LlamaParse to Parse Presentations\n",
    "\n",
@@ -127,14 +102,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "d3qeuiyawT0U",
-    "outputId": "cec0ea0a-be8b-49b6-9376-797c91f63be7",
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir data; wget \"https://meetings.wmo.int/Cg-19/PublishingImages/SitePages/FINAC-43/7%20-%20EC-77-Doc%205%20Financial%20Statements%20for%202022%20(FINAC).pptx\" -O data/presentation.pptx"
@@ -142,9 +110,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "Gbr8RiHEyF3-"
-   },
+   "metadata": {},
   "source": [
    "### Parsing the presentation\n",
    "\n",
@@ -155,24 +121,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "osocsofoJ42S"
-   },
+   "metadata": {},
   "source": [
    "#### Llama Index default"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "PTVy5XCNJwW-",
-    "outputId": "d0e2cc4b-1407-45a9-b5e6-d06f91a533b4",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import SimpleDirectoryReader\n",
@@ -182,24 +139,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "oucbsciZJwxt"
-   },
+   "metadata": {},
   "source": [
    "#### Llama Parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "3jKnXCuAyQ9_",
-    "outputId": "1f668f17-1e20-46e5-fbab-9a55e4b28891",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -210,7 +158,9 @@
    }
   ],
   "source": [
-    "llama_parse_documents = LlamaParse(result_type=\"markdown\").load_data(\"./data/presentation.pptx\")"
+    "llama_parse_documents = LlamaParse(result_type=\"markdown\").load_data(\n",
+    "    \"./data/presentation.pptx\"\n",
+    ")"
   ]
  },
  {
@@ -224,10 +174,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -259,9 +207,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "source": [
    "Compared against the original slide image.\n",
    "![Demo](demo_ppt_financial_1.png)"
@@ -269,9 +215,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "p4GVOdWzzvYg"
-   },
+   "metadata": {},
   "source": [
    "## Comparing the two for RAG\n",
    "\n",
@@ -280,20 +224,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "oVcdGus5NDxi"
-   },
+   "metadata": {},
   "source": [
    "### Query Engine on SimpleDirectoryReader results"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {
-    "id": "DqXYsLCWNg9_",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
@@ -304,20 +243,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "ZLkHt9l2Nbxx"
-   },
+   "metadata": {},
   "source": [
    "### Query Engine on LlamaParse Results\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {
-    "id": "ZllaDcfRNLv3",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "llama_parse_index = VectorStoreIndex.from_documents(llama_parse_documents)\n",
@@ -326,10 +260,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "0dY_0_1bNg0X",
-    "tags": []
-   },
+   "metadata": {},
   "source": [
    "### Liability provision\n",
    "What was the liability provision as of Dec 31 2021?\n",
@@ -339,15 +270,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "Tmn-qNTEN-cb",
-    "outputId": "a9bffc00-9cfc-43d8-b159-596a6c1aca64",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -358,21 +282,16 @@
    }
   ],
   "source": [
-    "vanilla_response = vanilla_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
+    "vanilla_response = vanilla_query_engine.query(\n",
+    "    \"What was the liability provision as of Dec 31 2021?\"\n",
+    ")\n",
    "print(vanilla_response)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "4EZ_uqlROP7R",
-    "outputId": "0645a159-06c6-411e-d1f6-79ea95d32b42",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -383,16 +302,11 @@
    }
   ],
   "source": [
-    "llama_parse_response = llama_parse_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
+    "llama_parse_response = llama_parse_query_engine.query(\n",
+    "    \"What was the liability provision as of Dec 31 2021?\"\n",
+    ")\n",
    "print(llama_parse_response)"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -413,8 +327,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -1,3 +1,3 @@
 from llama_parse.base import LlamaParse, ResultType

-__all__ = ["LlamaParse", "ResultType"]
+__all__ = ["LlamaParse", "ResultType"]
@@ -3,7 +3,6 @@ import asyncio
 import httpx
 import mimetypes
 import time
-from enum import Enum
 from pathlib import Path
 from typing import List, Optional, Union

@@ -12,113 +11,13 @@ from llama_index.core.bridge.pydantic import Field, validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
 from llama_index.core.schema import Document
-
-
-nest_asyncio_err = "cannot be called from a running event loop"
-nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
-
-class ResultType(str, Enum):
-    """The result type for the parser."""
-    TXT = "text"
-    MD = "markdown"
-
-class Language(str, Enum):
-    BAZA = "abq"
-    ADYGHE = "ady"
-    AFRIKAANS = "af"
-    ANGIKA = "ang"
-    ARABIC = "ar"
-    ASSAMESE = "as"
-    AVAR = "ava"
-    AZERBAIJANI = "az"
-    BELARUSIAN = "be"
-    BULGARIAN = "bg"
-    BIHARI = "bh"
-    BHOJPURI = "bho"
-    BENGALI = "bn"
-    BOSNIAN = "bs"
-    SIMPLIFIED_CHINESE = "ch_sim"
-    TRADITIONAL_CHINESE = "ch_tra"
-    CHECHEN = "che"
-    CZECH = "cs"
-    WELSH = "cy"
-    DANISH = "da"
-    DARGWA = "dar"
-    GERMAN = "de"
-    ENGLISH = "en"
-    SPANISH = "es"
-    ESTONIAN = "et"
-    PERSIAN_FARSI = "fa"
-    FRENCH = "fr"
-    IRISH = "ga"
-    GOAN_KONKANI = "gom"
-    HINDI = "hi"
-    CROATIAN = "hr"
-    HUNGARIAN = "hu"
-    INDONESIAN = "id"
-    INGUSH = "inh"
-    ICELANDIC = "is"
-    ITALIAN = "it"
-    JAPANESE = "ja"
-    KABARDIAN = "kbd"
-    KANNADA = "kn"
-    KOREAN = "ko"
-    KURDISH = "ku"
-    LATIN = "la"
-    LAK = "lbe"
-    LEZGHIAN = "lez"
-    LITHUANIAN = "lt"
-    LATVIAN = "lv"
-    MAGAHI = "mah"
-    MAITHILI = "mai"
-    MAORI = "mi"
-    MONGOLIAN = "mn"
-    MARATHI = "mr"
-    MALAY = "ms"
-    MALTESE = "mt"
-    NEPALI = "ne"
-    NEWARI = "new"
-    DUTCH = "nl"
-    NORWEGIAN = "no"
-    OCCITAN = "oc"
-    PALI = "pi"
-    POLISH = "pl"
-    PORTUGUESE = "pt"
-    ROMANIAN = "ro"
-    RUSSIAN = "ru"
-    SERBIAN_CYRILLIC = "rs_cyrillic"
-    SERBIAN_LATIN = "rs_latin"
-    NAGPURI = "sck"
-    SLOVAK = "sk"
-    SLOVENIAN = "sl"
-    ALBANIAN = "sq"
-    SWEDISH = "sv"
-    SWAHILI = "sw"
-    TAMIL = "ta"
-    TABASSARAN = "tab"
-    TELUGU = "te"
-    THAI = "th"
-    TAJIK = "tjk"
-    TAGALOG = "tl"
-    TURKISH = "tr"
-    UYGHUR = "ug"
-    UKRANIAN = "uk"
-    URDU = "ur"
-    UZBEK = "uz"
-    VIETNAMESE = "vi"
-
-
-SUPPORTED_FILE_TYPES = [
-    ".pdf",
-    ".xml"
-    ".doc",
-    ".docx",
-    ".pptx",
-    ".rtf",
-    ".pages",
-    ".key",
-    ".epub"
-]
+from llama_parse.utils import (
+    nest_asyncio_err,
+    nest_asyncio_msg,
+    ResultType,
+    Language,
+    SUPPORTED_FILE_TYPES,
+)


 class LlamaParse(BasePydanticReader):
@@ -135,8 +34,8 @@ class LlamaParse(BasePydanticReader):
    num_workers: int = Field(
        default=4,
        gt=0,
-        lt=10, 
-        description="The number of workers to use sending API requests for parsing."
+        lt=10,
+        description="The number of workers to use sending API requests for parsing.",
    )
    check_interval: int = Field(
        default=1,
@@ -149,12 +48,18 @@ class LlamaParse(BasePydanticReader):
    verbose: bool = Field(
        default=True, description="Whether to print the progress of the parsing."
    )
-    language: Language  = Field(
-         default=Language.ENGLISH, description="The language of the text to parse."
+    show_progress: bool = Field(
+        default=True, description="Show progress when parsing multiple files."
+    )
+    language: Language = Field(
+        default=Language.ENGLISH, description="The language of the text to parse."
    )
    parsing_instruction: Optional[str] = Field(
-        default="",
-        description="The parsing instruction for the parser."
+        default="", description="The parsing instruction for the parser."
+    )
+    ignore_errors: bool = Field(
+        default=True,
+        description="Whether or not to ignore and skip errors raised during parsing.",
    )

    @validator("api_key", pre=True, always=True)
@@ -162,13 +67,14 @@ class LlamaParse(BasePydanticReader):
        """Validate the API key."""
        if not v:
            import os
+
            api_key = os.getenv("LLAMA_CLOUD_API_KEY", None)
            if api_key is None:
                raise ValueError("The API key is required.")
            return api_key
-        
+
        return v
-    
+
    @validator("base_url", pre=True, always=True)
    def validate_base_url(cls, v: str) -> str:
        """Validate the base URL."""
@@ -176,7 +82,9 @@ class LlamaParse(BasePydanticReader):
        return url or v or DEFAULT_BASE_URL

    # upload a document and get back a job_id
-    async def _create_job(self, file_path: str, extra_info: Optional[dict] = None) -> str:
+    async def _create_job(
+        self, file_path: str, extra_info: Optional[dict] = None
+    ) -> str:
        file_path = str(file_path)
        file_ext = os.path.splitext(file_path)[1]
        if file_ext not in SUPPORTED_FILE_TYPES:
@@ -198,7 +106,15 @@ class LlamaParse(BasePydanticReader):
            # send the request, start job
            url = f"{self.base_url}/api/parsing/upload"
            async with httpx.AsyncClient(timeout=self.max_timeout) as client:
-                response = await client.post(url, files=files, headers=headers, data={"language": self.language.value, "parsing_instruction": self.parsing_instruction})
+                response = await client.post(
+                    url,
+                    files=files,
+                    headers=headers,
+                    data={
+                        "language": self.language.value,
+                        "parsing_instruction": self.parsing_instruction,
+                    },
+                )
                if not response.is_success:
                    raise Exception(f"Failed to parse the file: {response.text}")

@@ -206,43 +122,65 @@ class LlamaParse(BasePydanticReader):
        job_id = response.json()["id"]
        return job_id

-    async def _get_job_result(self, job_id: str, result_type: str) -> dict:
+    async def _get_job_result(
+        self, job_id: str, result_type: str, verbose: bool = False
+    ) -> dict:
        result_url = f"{self.base_url}/api/parsing/job/{job_id}/result/{result_type}"
+        status_url = f"{self.base_url}/api/parsing/job/{job_id}"
        headers = {"Authorization": f"Bearer {self.api_key}"}

        start = time.time()
        tries = 0
        while True:
            await asyncio.sleep(self.check_interval)
-            async with httpx.AsyncClient(timeout=self.max_timeout) as client: 
-                tries += 1   
-                
-                result = await client.get(result_url, headers=headers)
+            async with httpx.AsyncClient(timeout=self.max_timeout) as client:
+                tries += 1

-                if result.status_code == 404:
+                result = await client.get(status_url, headers=headers)
+
+                if result.status_code != 200:
                    end = time.time()
                    if end - start > self.max_timeout:
-                        raise Exception(
-                            f"Timeout while parsing the file: {job_id}"
-                        )
-                    if self.verbose and tries % 10 == 0:
+                        raise Exception(f"Timeout while parsing the file: {job_id}")
+                    if verbose and tries % 10 == 0:
                        print(".", end="", flush=True)
+
+                    await asyncio.sleep(self.check_interval)
+
                    continue

-                if result.status_code == 400:
-                    detail = result.json().get("detail", "Unknown error")
-                    raise Exception(f"Failed to parse the file: {detail}")
+                # Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
+                status = result.json()["status"]
+                if status == "SUCCESS":
+                    parsed_result = await client.get(result_url, headers=headers)
+                    return parsed_result.json()
+                elif status == "PENDING":
+                    end = time.time()
+                    if end - start > self.max_timeout:
+                        raise Exception(f"Timeout while parsing the file: {job_id}")
+                    if verbose and tries % 10 == 0:
+                        print(".", end="", flush=True)

-                return result.json()
+                    await asyncio.sleep(self.check_interval)

-    async def _aload_data(self, file_path: str, extra_info: Optional[dict] = None) -> List[Document]:
+                    continue
+                else:
+                    raise Exception(
+                        f"Failed to parse the file: {job_id}, status: {status}"
+                    )
+
+    async def _aload_data(
+        self, file_path: str, extra_info: Optional[dict] = None, verbose: bool = False
+    ) -> List[Document]:
        """Load data from the input path."""
        try:
            job_id = await self._create_job(file_path, extra_info=extra_info)
-            if self.verbose:
+            if verbose:
                print("Started parsing the file under job_id %s" % job_id)
-            
-            result = await self._get_job_result(job_id, self.result_type.value)
+
+            result = await self._get_job_result(
+                job_id, self.result_type.value, verbose=verbose
+            )

            return [
                Document(
@@ -250,22 +188,39 @@ class LlamaParse(BasePydanticReader):
                    metadata=extra_info or {},
                )
            ]
-          
+
        except Exception as e:
            print(f"Error while parsing the file '{file_path}':", e)
-            raise e
-            return []
-    
+            if self.ignore_errors:
+                return []
+            else:
+                raise e

-    async def aload_data(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[Document]:
+    async def aload_data(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[Document]:
        """Load data from the input path."""
        if isinstance(file_path, (str, Path)):
-            return await self._aload_data(file_path, extra_info=extra_info)
+            return await self._aload_data(
+                file_path, extra_info=extra_info, verbose=self.verbose
+            )
        elif isinstance(file_path, list):
-            jobs = [self._aload_data(f, extra_info=extra_info) for f in file_path]
+            jobs = [
+                self._aload_data(
+                    f,
+                    extra_info=extra_info,
+                    verbose=self.verbose and not self.show_progress,
+                )
+                for f in file_path
+            ]
            try:
-                results = await run_jobs(jobs, workers=self.num_workers)
-                
+                results = await run_jobs(
+                    jobs,
+                    workers=self.num_workers,
+                    desc="Parsing files",
+                    show_progress=self.show_progress,
+                )
+
                # return flattened results
                return [item for sublist in results for item in sublist]
            except RuntimeError as e:
@@ -274,9 +229,13 @@ class LlamaParse(BasePydanticReader):
                else:
                    raise e
        else:
-            raise ValueError("The input file_path must be a string or a list of strings.")
+            raise ValueError(
+                "The input file_path must be a string or a list of strings."
+            )

-    def load_data(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[Document]:
+    def load_data(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[Document]:
        """Load data from the input path."""
        try:
            return asyncio.run(self.aload_data(file_path, extra_info))
@@ -285,35 +244,44 @@ class LlamaParse(BasePydanticReader):
                raise RuntimeError(nest_asyncio_msg)
            else:
                raise e
-    

-    async def _aget_json(self, file_path: str, extra_info: Optional[dict] = None) -> List[dict]:
+    async def _aget_json(
+        self, file_path: str, extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Load data from the input path."""
        try:
            job_id = await self._create_job(file_path, extra_info=extra_info)
            if self.verbose:
                print("Started parsing the file under job_id %s" % job_id)
-            
+
            result = await self._get_job_result(job_id, "json")
            result["job_id"] = job_id
            result["file_path"] = file_path
            return [result]
-          
+
        except Exception as e:
            print(f"Error while parsing the file '{file_path}':", e)
-            raise e
-        
-    
+            if self.ignore_errors:
+                return []
+            else:
+                raise e

-    async def aget_json(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[dict]:
+    async def aget_json(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Load data from the input path."""
        if isinstance(file_path, (str, Path)):
            return await self._aget_json(file_path, extra_info=extra_info)
        elif isinstance(file_path, list):
            jobs = [self._aget_json(f, extra_info=extra_info) for f in file_path]
            try:
-                results = await run_jobs(jobs, workers=self.num_workers)
-                
+                results = await run_jobs(
+                    jobs,
+                    workers=self.num_workers,
+                    desc="Parsing files",
+                    show_progress=self.show_progress,
+                )
+
                # return flattened results
                return [item for sublist in results for item in sublist]
            except RuntimeError as e:
@@ -322,10 +290,13 @@ class LlamaParse(BasePydanticReader):
                else:
                    raise e
        else:
-            raise ValueError("The input file_path must be a string or a list of strings.")
+            raise ValueError(
+                "The input file_path must be a string or a list of strings."
+            )

-
-    def get_json_result(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[dict]:
+    def get_json_result(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Parse the input path."""
        try:
            return asyncio.run(self.aget_json(file_path, extra_info))
@@ -334,10 +305,15 @@ class LlamaParse(BasePydanticReader):
                raise RuntimeError(nest_asyncio_msg)
            else:
                raise e
-            
-    def get_images(self, json_result: list[dict], download_path: str) -> List[dict]:
+
+    def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
        """Download images from the parsed result."""
        headers = {"Authorization": f"Bearer {self.api_key}"}
+
+        # make the download path
+        if not os.path.exists(download_path):
+            os.makedirs(download_path)
+
        try:
            images = []
            for result in json_result:
@@ -347,16 +323,28 @@ class LlamaParse(BasePydanticReader):
                        print(f"> Image for page {page['page']}: {page['images']}")
                    for image in page["images"]:
                        image_name = image["name"]
-                        image_path = os.path.join(download_path, f"{job_id}-{image_name}")
-                        image["path"]=image_path
-                        image["job_id"]=job_id
-                        image["original_pdf_path"]=result["file_path"]
-                        image["page_number"]=page["page"]
+
+                        # get the full path
+                        image_path = os.path.join(
+                            download_path, f"{job_id}-{image_name}"
+                        )
+
+                        # get a valid image path
+                        if not image_path.endswith(".png"):
+                            image_path += ".png"
+
+                        image["path"] = image_path
+                        image["job_id"] = job_id
+                        image["original_pdf_path"] = result["file_path"]
+                        image["page_number"] = page["page"]
                        with open(image_path, "wb") as f:
                            image_url = f"{self.base_url}/api/parsing/job/{job_id}/result/image/{image_name}"
                            f.write(httpx.get(image_url, headers=headers).content)
                        images.append(image)
            return images
        except Exception as e:
-            print(f"Error while downloading images from the parsed result:", e)
-            return []
+            print("Error while downloading images from the parsed result:", e)
+            if self.ignore_errors:
+                return []
+            else:
+                raise e
@@ -0,0 +1,152 @@
+from enum import Enum
+
+# Asyncio error messages
+nest_asyncio_err = "cannot be called from a running event loop"
+nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
+
+
+class ResultType(str, Enum):
+    """The result type for the parser."""
+
+    TXT = "text"
+    MD = "markdown"
+    JSON = "json"
+
+
+class Language(str, Enum):
+    BAZA = "abq"
+    ADYGHE = "ady"
+    AFRIKAANS = "af"
+    ANGIKA = "ang"
+    ARABIC = "ar"
+    ASSAMESE = "as"
+    AVAR = "ava"
+    AZERBAIJANI = "az"
+    BELARUSIAN = "be"
+    BULGARIAN = "bg"
+    BIHARI = "bh"
+    BHOJPURI = "bho"
+    BENGALI = "bn"
+    BOSNIAN = "bs"
+    SIMPLIFIED_CHINESE = "ch_sim"
+    TRADITIONAL_CHINESE = "ch_tra"
+    CHECHEN = "che"
+    CZECH = "cs"
+    WELSH = "cy"
+    DANISH = "da"
+    DARGWA = "dar"
+    GERMAN = "de"
+    ENGLISH = "en"
+    SPANISH = "es"
+    ESTONIAN = "et"
+    PERSIAN_FARSI = "fa"
+    FRENCH = "fr"
+    IRISH = "ga"
+    GOAN_KONKANI = "gom"
+    HINDI = "hi"
+    CROATIAN = "hr"
+    HUNGARIAN = "hu"
+    INDONESIAN = "id"
+    INGUSH = "inh"
+    ICELANDIC = "is"
+    ITALIAN = "it"
+    JAPANESE = "ja"
+    KABARDIAN = "kbd"
+    KANNADA = "kn"
+    KOREAN = "ko"
+    KURDISH = "ku"
+    LATIN = "la"
+    LAK = "lbe"
+    LEZGHIAN = "lez"
+    LITHUANIAN = "lt"
+    LATVIAN = "lv"
+    MAGAHI = "mah"
+    MAITHILI = "mai"
+    MAORI = "mi"
+    MONGOLIAN = "mn"
+    MARATHI = "mr"
+    MALAY = "ms"
+    MALTESE = "mt"
+    NEPALI = "ne"
+    NEWARI = "new"
+    DUTCH = "nl"
+    NORWEGIAN = "no"
+    OCCITAN = "oc"
+    PALI = "pi"
+    POLISH = "pl"
+    PORTUGUESE = "pt"
+    ROMANIAN = "ro"
+    RUSSIAN = "ru"
+    SERBIAN_CYRILLIC = "rs_cyrillic"
+    SERBIAN_LATIN = "rs_latin"
+    NAGPURI = "sck"
+    SLOVAK = "sk"
+    SLOVENIAN = "sl"
+    ALBANIAN = "sq"
+    SWEDISH = "sv"
+    SWAHILI = "sw"
+    TAMIL = "ta"
+    TABASSARAN = "tab"
+    TELUGU = "te"
+    THAI = "th"
+    TAJIK = "tjk"
+    TAGALOG = "tl"
+    TURKISH = "tr"
+    UYGHUR = "ug"
+    UKRAINIAN = "uk"
+    URDU = "ur"
+    UZBEK = "uz"
+    VIETNAMESE = "vi"
+
+
+SUPPORTED_FILE_TYPES = [
+    ".pdf",
+    # Microsoft word - all versions
+    ".doc",
+    ".docx",
+    ".docm",
+    ".dot",
+    ".dotx",
+    ".dotm",
+    # Rich text format
+    ".rtf",
+    # Microsoft Works
+    ".wps",
+    # Word Perfect
+    ".wpd",
+    # Open Office
+    ".sxw",
+    ".stw",
+    ".sxg",
+    # Apple
+    ".pages",
+    # Mac Write
+    ".mw",
+    ".mcw",
+    # Unified Office Format text
+    ".uot",
+    ".uof",
+    ".uos",
+    ".uop",
+    # Microsoft powerpoints
+    ".ppt",
+    ".pptx",
+    ".pot",
+    ".pptm",
+    ".potx",
+    ".potm",
+    # Apple keynote
+    ".key",
+    # Open Office Presentations
+    ".odp",
+    ".odg",
+    ".otp",
+    ".fopd",
+    ".sxi",
+    ".sti",
+    # ebook
+    ".epub",
+    # html
+    ".html",
+    ".htm",
+]
@@ -1,6 +1,10 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "llama-parse"
-version = "0.3.9"
+version = "0.4.2"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"
@@ -9,12 +13,8 @@ packages = [{include = "llama_parse"}]

 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
-llama-index-core = ">=0.10.7"
+llama-index-core = ">=0.10.29"

 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
 ipykernel = "^6.29.0"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
@@ -1,18 +1,59 @@
 import os
+import pytest
 from llama_parse import LlamaParse

-def test_simple_page_text():
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_text() -> None:
    parser = LlamaParse(result_type="text")

-    filepath = os.path.join(os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf")
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
    result = parser.load_data(filepath)
    assert len(result) == 1
    assert len(result[0].text) > 0

-def test_simple_page_markdown():
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_markdown() -> None:
    parser = LlamaParse(result_type="markdown")

-    filepath = os.path.join(os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf")
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
    result = parser.load_data(filepath)
    assert len(result) == 1
    assert len(result[0].text) > 0
+
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_progress_workers() -> None:
+    parser = LlamaParse(result_type="markdown", show_progress=True, verbose=True)
+
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
+    result = parser.load_data([filepath, filepath])
+    assert len(result) == 2
+    assert len(result[0].text) > 0
+
+    parser = LlamaParse(
+        result_type="markdown", show_progress=True, num_workers=2, verbose=True
+    )
+
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
+    result = parser.load_data([filepath, filepath])
+    assert len(result) == 2
+    assert len(result[0].text) > 0
Author	SHA1	Message	Date
Logan	9ed208131f	Ensure image extensions, vbump (#159 )	2024-04-24 20:39:39 -06:00
Pierre-Loic Doulcet	91b03b2ea7	add html support (#154 )	2024-04-23 13:14:13 -06:00
Yi Ding	70b5dc3a63	some notebook updates (#148 )	2024-04-18 21:57:10 -06:00
Anoop Sharma	d6ab0aa232	Added utils files (#91 )	2024-04-14 15:17:31 -06:00
Logan	f679e1c76b	Skip tests if CICD doesn't populate the secrets (#142 )	2024-04-14 15:12:22 -06:00
Logan	b91f86ba3d	v0.4.1 (#141 )	2024-04-14 13:09:00 -06:00
Logan	0f2302fda4	QoL Changes (#140 )	2024-04-14 13:01:39 -06:00
henrycunh	ff729c05af	docs: update readme with link to llamacloud (#139 ) Having copy paste a link is silly!	2024-04-14 11:56:47 -07:00
Gautam Kumar	76a6821fb8	Fixing paths of data in example notebook (#136 )	2024-04-10 15:52:35 -06:00
Logan	97c7a38a69	vbump (#111 )	2024-03-21 10:51:50 -06:00
Pierre-Loic Doulcet	5d398a8a64	Extend supported formats (#110 )	2024-03-21 10:44:18 -06:00
Jerry Liu	4252f6186b	fixes to insurance demo (#97 )	2024-03-21 00:02:47 -07:00
Jerry Liu	22148ade9f	fix advanced RAG notebook title (#98 )	2024-03-21 00:02:38 -07:00
Jerry Liu	b8332fe8e1	nit: add colab badge to mongodb notebook (#109 )	2024-03-21 00:02:29 -07:00
Ravi Theja	e40e92a133	Add mongodb llamaparse example (#107 )	2024-03-20 23:37:42 -07:00
Jerry Liu	ba8f345f80	Revert "cr" This reverts commit `2ddbf1ba0d`.	2024-03-19 00:21:29 -07:00
Jerry Liu	2ddbf1ba0d	cr	2024-03-19 00:20:42 -07:00
Haotian Zhang	23567c8f98	Init LlamaParseJsonNodeParser example (#93 )	2024-03-18 15:15:36 -04:00
Logan	8d39ae7763	add agent demo (#88 ) * add agent demo * remove mention of react agent * agents folder	2024-03-18 16:12:56 +01:00
Jerry Liu	a2edc41fc7	nit: fix grammar in insurance cookbook (#89 ) cr	2024-03-18 16:11:03 +01:00
Ikko Eltociear Ashimine	591b6fc44d	Update demo_parsing_instructions.ipynb (#86 ) usefull -> useful	2024-03-16 18:15:30 +01:00
Pierre-Loic Doulcet	f8a3d92ce0	demo insurance + parsing instructions (#84 )	2024-03-16 18:14:49 +01:00
Laurie Voss	a1d18d83da	Adding parsing instructions demo (#82 )	2024-03-15 10:07:53 +01:00