add html support

some notebook updates (#148 )
Added utils files (#91 )
2026-07-21 12:05:23 -04:00 · 2024-04-23 11:21:27 +08:00 · 2024-04-18 21:57:10 -06:00 · 2024-04-14 15:17:31 -06:00 · 2024-04-14 15:12:22 -06:00 · 2024-04-14 13:09:00 -06:00
30 changed files with 2508 additions and 2047 deletions
@@ -0,0 +1,48 @@
+name: Build Package
+
+# Build package on its own without additional pip install
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: poetry install
+      - name: Ensure lock works
+        shell: bash
+        run: poetry lock
+      - name: Build
+        shell: bash
+        run: poetry build
+      - name: Test installing built package
+        shell: bash
+        run: python -m pip install .
+      - name: Test import
+        shell: bash
+        working-directory: ${{ vars.RUNNER_TEMP }}
+        run: python -c "import llama_parse"
@@ -0,0 +1,81 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: ["main"]
+  schedule:
+    - cron: "30 16 * * 4"
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["python"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+
+          # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
+
+      # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+      # - run: |
+      #     echo "Run, Build Application using script"
+      #     ./location_of_script_within_repo/buildscript.sh
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{matrix.language}}"
@@ -0,0 +1,37 @@
+name: Linting
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install pre-commit
+        shell: bash
+        run: poetry run pip install pre-commit
+      - name: Run linter
+        shell: bash
+        run: poetry run make lint
@@ -0,0 +1,64 @@
+name: Publish llama-parse to PyPI / GitHub
+
+on:
+  push:
+    tags:
+      - "v*"
+
+  workflow_dispatch:
+
+env:
+  POETRY_VERSION: "1.6.1"
+  PYTHON_VERSION: "3.9"
+
+jobs:
+  build-n-publish:
+    name: Build and publish to PyPI
+    if: github.repository == 'run-llama/llama_parse'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: pip install -e .
+      - name: Build and publish to pypi
+        uses: JRubics/poetry-publish@v1.17
+        with:
+          pypi_token: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
+          ignore_dev_requirements: "yes"
+
+      - name: Create GitHub Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Get Asset name
+        run: |
+          export PKG=$(ls dist/ | grep tar)
+          set -- $PKG
+          echo "name=$1" >> $GITHUB_ENV
+      - name: Upload Release Asset (sdist) to GitHub
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: dist/${{ env.name }}
+          asset_name: ${{ env.name }}
+          asset_content_type: application/zip
@@ -0,0 +1,40 @@
+name: Unit Testing
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.6.1"
+  LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      # You can use PyPy versions in python-version.
+      # For example, pypy-2.7 and pypy-3.8
+      matrix:
+        python-version: ["3.8", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: poetry install --with dev
+      - name: Run testing
+        env:
+          CI: true
+        shell: bash
+        run: poetry run pytest tests
@@ -1,4 +1,4 @@
 .git
 __pycache__/
 *.pyc
-.DS_Store
+.DS_Store
@@ -0,0 +1,86 @@
+---
+default_language_version:
+  python: python3
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-byte-order-marker
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.1.5
+
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        exclude: ".*poetry.lock"
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.10.1
+    hooks:
+      - id: black-jupyter
+        name: black-src
+        alias: black
+        exclude: ".*poetry.lock"
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.0.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [
+            "types-requests",
+            "types-Deprecated",
+            "types-redis",
+            "types-setuptools",
+            "types-PyYAML",
+            "types-protobuf==4.24.0.4",
+          ]
+        args:
+          [
+            --disallow-untyped-defs,
+            --ignore-missing-imports,
+            --python-version=3.8,
+          ]
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+      - id: blacken-docs
+        name: black-docs-text
+        alias: black
+        types_or: [rst, markdown, tex]
+        additional_dependencies: [black==23.10.1]
+        # Using PEP 8's line length in docs prevents excess left/right scrolling
+        args: [--line-length=79]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier
+        exclude: poetry.lock
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies: [tomli]
+        exclude: ^(poetry.lock|examples)
+        args:
+          [
+            "--ignore-words-list",
+            "astroid,gallary,momento,narl,ot,rouge,nin,gere,te,inh",
+          ]
+  - repo: https://github.com/srstevenson/nb-clean
+    rev: 3.1.0
+    hooks:
+      - id: nb-clean
+        args: [--preserve-cell-outputs, --remove-empty-cells]
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
+        exclude: ".*poetry.lock"
@@ -0,0 +1,14 @@
+GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
+
+help:	## Show all Makefile targets.
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+
+format:	## Run code autoformatters (black).
+	pre-commit install
+	git ls-files | xargs pre-commit run black --files
+
+lint:	## Run linters: pre-commit (black, ruff, codespell) and mypy
+	pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+
+test:	## Run tests via pytest
+	pytest tests
@@ -4,12 +4,11 @@ LlamaParse is an API created by LlamaIndex to efficiently parse and represent fi

 LlamaParse directly integrates with [LlamaIndex](https://github.com/run-llama/llama_index).

-
 Free plan is up to 1000 pages a day. Paid plan is free 7k pages per week + 0.3c per additional page.

 ## Getting Started

-First, login and get an api-key from `https://cloud.llamaindex.ai`.
+First, login and get an api-key from [**https://cloud.llamaindex.ai ↗**](https://cloud.llamaindex.ai).

 Then, make sure you have the latest LlamaIndex version installed.

@@ -28,6 +27,7 @@ Now you can run the following to parse your first PDF file:

 ```python
 import nest_asyncio
+
 nest_asyncio.apply()

 from llama_parse import LlamaParse
@@ -35,9 +35,9 @@ from llama_parse import LlamaParse
 parser = LlamaParse(
    api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
    result_type="markdown",  # "markdown" and "text" are available
-    num_workers=4, # if multiple files passed, split in `num_workers` API calls
+    num_workers=4,  # if multiple files passed, split in `num_workers` API calls
    verbose=True,
-    language="en" # Optionaly you can define a language, default=en
+    language="en",  # Optionally you can define a language, default=en
 )

 # sync
@@ -59,6 +59,7 @@ You can also integrate the parser as the default PDF loader in `SimpleDirectoryR

 ```python
 import nest_asyncio
+
 nest_asyncio.apply()

 from llama_parse import LlamaParse
@@ -67,11 +68,13 @@ from llama_index.core import SimpleDirectoryReader
 parser = LlamaParse(
    api_key="llx-...",  # can also be set in your env as LLAMA_CLOUD_API_KEY
    result_type="markdown",  # "markdown" and "text" are available
-    verbose=True
+    verbose=True,
 )

 file_extractor = {".pdf": parser}
-documents = SimpleDirectoryReader("./data", file_extractor=file_extractor).load_data()
+documents = SimpleDirectoryReader(
+    "./data", file_extractor=file_extractor
+).load_data()
 ```

 Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex Documentation](https://docs.llamaindex.ai/en/stable/module_guides/loading/simpledirectoryreader.html).
@@ -27,7 +27,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -78,7 +78,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -91,7 +91,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -108,7 +108,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -129,9 +129,13 @@
   ],
   "source": [
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
-    "from llama_index.core.node_parser import MarkdownElementNodeParser, SentenceSplitter\n",
+    "from llama_index.core.node_parser import (\n",
+    "    MarkdownElementNodeParser,\n",
+    "    SentenceSplitter,\n",
+    ")\n",
    "\n",
    "# explicitly extract tables with the MarkdownElementNodeParser\n",
    "node_parser = MarkdownElementNodeParser(num_workers=8)\n",
@@ -139,7 +143,9 @@
    "nodes, objects = node_parser.get_nodes_and_objects(nodes)\n",
    "\n",
    "# Chain splitters to ensure chunk size requirements are met\n",
-    "nodes = SentenceSplitter(chunk_size=512, chunk_overlap=20).get_nodes_from_documents(nodes)"
+    "nodes = SentenceSplitter(chunk_size=512, chunk_overlap=20).get_nodes_from_documents(\n",
+    "    nodes\n",
+    ")"
   ]
  },
  {
@@ -151,7 +157,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -163,7 +169,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -174,8 +180,7 @@
    "tools = [\n",
    "    QueryEngineTool(\n",
    "        vector_index.as_query_engine(\n",
-    "            similarity_top_k=8,\n",
-    "            node_postprocessors=[ColbertRerank(top_n=3)]\n",
+    "            similarity_top_k=8, node_postprocessors=[ColbertRerank(top_n=3)]\n",
    "        ),\n",
    "        metadata=ToolMetadata(\n",
    "            name=\"search\",\n",
@@ -191,15 +196,12 @@
    "    ),\n",
    "]\n",
    "\n",
-    "agent = OpenAIAgent.from_tools(\n",
-    "    tools=tools, \n",
-    "    verbose=True\n",
-    ")"
+    "agent = OpenAIAgent.from_tools(tools=tools, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -222,7 +224,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -239,7 +241,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -261,7 +263,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -292,10 +294,8 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  },
-  "orig_nbformat": 4
+   "pygments_lexer": "ipython3"
+  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
@@ -4,9 +4,11 @@
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "# Llama Parser <> LlamaIndex\n",
+    "# Advanced RAG with LlamaParse\n",
    "\n",
-    "This notebook is a complete walkthrough for using `LlamaParse` for RAG applications with `LlamaIndex`.\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "This notebook shows you how to use LlamaParse with our advanced markdown ingestion and recursive retrieval algorithms to model tables/text within a document hierarchically. This lets you ask questions over both tables and text.\n",
    "\n",
    "Note for this example, we are using the `llama_index >=0.10.4` version"
   ]
@@ -17,12 +19,12 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
   ]
  },
  {
@@ -31,7 +33,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -43,15 +45,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -61,7 +65,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -70,11 +74,11 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
-    "Settings.embed_model = embed_model\n"
+    "Settings.embed_model = embed_model"
   ]
  },
  {
@@ -90,7 +94,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -104,12 +108,12 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "documents = LlamaParse(result_type=\"markdown\").load_data('./uber_10q_march_2022.pdf')"
+    "documents = LlamaParse(result_type=\"markdown\").load_data(\"./uber_10q_march_2022.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -144,23 +148,25 @@
    }
   ],
   "source": [
-    "print(documents[0].text[:1000] + '...')"
+    "print(documents[0].text[:1000] + \"...\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -185,7 +191,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -194,21 +200,23 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "recursive_index = VectorStoreIndex(nodes=base_nodes + objects)\n",
    "raw_index = VectorStoreIndex.from_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
@@ -216,17 +224,17 @@
    ")\n",
    "\n",
    "recursive_query_engine = recursive_index.as_query_engine(\n",
-    "    similarity_top_k=15, \n",
-    "    node_postprocessors=[reranker], \n",
-    "    verbose=True\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker], verbose=True\n",
    ")\n",
    "\n",
-    "raw_query_engine = raw_index.as_query_engine(similarity_top_k=15, node_postprocessors=[reranker])"
+    "raw_query_engine = raw_index.as_query_engine(\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker]\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -258,7 +266,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -295,7 +303,7 @@
    "\n",
    "response_2 = recursive_query_engine.query(query)\n",
    "print(\"\\n***********New LlamaParse+ Recursive Retriever Query Engine***********\")\n",
-    "print(response_2)\n"
+    "print(response_2)"
   ]
  },
  {
@@ -312,7 +320,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -372,7 +380,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -428,7 +436,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -502,8 +510,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -17,13 +17,13 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core==0.10.6.post1\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse\n",
-    "!pip install llama-index-vector-stores-astra-db"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core==0.10.6.post1\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse\n",
+    "%pip install llama-index-vector-stores-astra-db"
   ]
  },
  {
@@ -32,7 +32,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -44,15 +44,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -66,7 +68,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -75,7 +77,7 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
@@ -93,7 +95,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -107,12 +109,12 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "documents = LlamaParse(result_type=\"markdown\").load_data('./uber_10q_march_2022.pdf')"
+    "documents = LlamaParse(result_type=\"markdown\").load_data(\"./uber_10q_march_2022.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -168,7 +170,7 @@
    }
   ],
   "source": [
-    "print(documents[0].text[:1000] + '...')"
+    "print(documents[0].text[:1000] + \"...\")"
   ]
  },
  {
@@ -180,29 +182,27 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.vector_stores.astra_db import AstraDBVectorStore\n",
    "\n",
-    "# define two storage classes representing two collections (to compare advanced approach vs. baseline) \n",
+    "# define two storage classes representing two collections (to compare advanced approach vs. baseline)\n",
    "\n",
    "astra_db_store_advanced = AstraDBVectorStore(\n",
    "    token=ASTRA_TOKEN,\n",
    "    api_endpoint=ASTRA_API_ENDPOINT,\n",
    "    namespace=ASTRA_NAMESPACE,\n",
    "    collection_name=\"astra_v_table_llamaparse_advanced\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")\n",
    "astra_db_store_base = AstraDBVectorStore(\n",
    "    token=ASTRA_TOKEN,\n",
    "    api_endpoint=ASTRA_API_ENDPOINT,\n",
    "    namespace=ASTRA_NAMESPACE,\n",
    "    collection_name=\"astra_v_table_llamaparse_base\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")"
   ]
  },
@@ -219,13 +219,15 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
@@ -239,7 +241,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -248,17 +250,23 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import StorageContext\n",
    "\n",
-    "storage_context_advanced = StorageContext.from_defaults(vector_store=astra_db_store_advanced)\n",
+    "storage_context_advanced = StorageContext.from_defaults(\n",
+    "    vector_store=astra_db_store_advanced\n",
+    ")\n",
    "storage_context_base = StorageContext.from_defaults(vector_store=astra_db_store_base)\n",
    "\n",
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects, storage_context=storage_context_advanced)\n",
-    "raw_index = VectorStoreIndex.from_documents(documents, storage_context=storage_context_base)"
+    "recursive_index = VectorStoreIndex(\n",
+    "    nodes=base_nodes + objects, storage_context=storage_context_advanced\n",
+    ")\n",
+    "raw_index = VectorStoreIndex.from_documents(\n",
+    "    documents, storage_context=storage_context_base\n",
+    ")"
   ]
  },
  {
@@ -267,7 +275,9 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
@@ -275,12 +285,12 @@
    ")\n",
    "\n",
    "recursive_query_engine = recursive_index.as_query_engine(\n",
-    "    similarity_top_k=15, \n",
-    "    node_postprocessors=[reranker], \n",
-    "    verbose=True\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker], verbose=True\n",
    ")\n",
    "\n",
-    "raw_query_engine = raw_index.as_query_engine(similarity_top_k=15, node_postprocessors=[reranker])"
+    "raw_query_engine = raw_index.as_query_engine(\n",
+    "    similarity_top_k=15, node_postprocessors=[reranker]\n",
+    ")"
   ]
  },
  {
@@ -300,7 +310,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -341,7 +351,7 @@
    "\n",
    "response_2 = recursive_query_engine.query(query)\n",
    "print(\"\\n***********New LlamaParse+ Recursive Retriever Query Engine***********\")\n",
-    "print(response_2)\n"
+    "print(response_2)"
   ]
  },
  {
@@ -358,7 +368,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -422,7 +432,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -478,7 +488,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -554,8 +564,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -11,7 +11,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -38,7 +38,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -47,7 +47,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -88,7 +88,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -128,10 +128,8 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  },
-  "orig_nbformat": 4
+   "pygments_lexer": "ipython3"
+  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
@@ -23,12 +23,12 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, install the required dependencies\n",
-    "!pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
+    "%pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
   ]
  },
  {
@@ -40,7 +40,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -53,7 +53,9 @@
    "llama_cloud_api_key = getpass(\"Enter your Llama Index Cloud API Key: \")\n",
    "api_endpoint = input(\"Enter your Astra DB API Endpoint: \")\n",
    "token = getpass(\"Enter your Astra DB Token: \")\n",
-    "namespace = input(\"Enter your Astra DB namespace (optional, must exist on Astra): \") or None\n",
+    "namespace = (\n",
+    "    input(\"Enter your Astra DB namespace (optional, must exist on Astra): \") or None\n",
+    ")\n",
    "openai_api_key = getpass(\"Enter your OpenAI API Key: \")\n",
    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = llama_cloud_api_key\n",
@@ -62,7 +64,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -81,7 +83,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -94,7 +96,7 @@
   ],
   "source": [
    "# Grab a PDF from Arxiv for indexing\n",
-    "import requests \n",
+    "import requests\n",
    "\n",
    "# The URL of the file you want to download\n",
    "url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
@@ -116,7 +118,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -135,7 +137,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -144,7 +146,7 @@
       "'rmer - model architecture.\\nThe Transformer follows this overall architecture using stacked self-attention and point-wise, fully\\nconnected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\\nrespectively.\\n3.1   Encoder and Decoder Stacks\\nEncoder:     The encoder is composed of a stack of N = 6 identical layers. Each layer has two\\nsub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\\nwise fully connected feed-forward network. We employ a residual connection [11] around each of\\nthe two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\\nLayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\\nitself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\\nlayers, produce outputs of dimension dmodel = 512.\\nDecoder:    The decoder is also composed of a stack of N = 6 identical layers. In addition '"
      ]
     },
-     "execution_count": 6,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -163,7 +165,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -174,16 +176,14 @@
    "    api_endpoint=api_endpoint,\n",
    "    namespace=namespace,\n",
    "    collection_name=\"astra_v_table_llamaparse\",\n",
-    "    embedding_dimension=1536\n",
+    "    embedding_dimension=1536,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import SimpleNodeParser\n",
@@ -195,7 +195,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -220,7 +220,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -229,7 +229,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -252,7 +252,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -261,7 +261,7 @@
       "'We used beam search as described in the previous section, but no\\ncheckpoint averaging. We present these results in Table 3.\\nIn Table 3 rows (A), we vary the number of attention heads and the attention key and value dimensions,\\nkeeping the amount of computation constant, as described in Section 3.2.2. While single-head\\nattention is 0.9 BLEU worse than the best setting, quality also drops off with too many heads.\\nIn Table 3 rows (B), we observe that reducing the attention key size dk hurts model quality. This\\nsuggests that determining compatibility is not easy and that a more sophisticated compatibility\\nfunction than dot product may be beneficial. We further observe in rows (C) and (D) that, as expected,\\nbigger models are better, and dropout is very helpful in avoiding over-fitting. In row (E) we replace our\\nsinusoidal positional encoding with learned positional embeddings [9], and observe nearly identical\\nresults to the base model.\\n6.3    English Constituency Parsing\\nTo evaluate if the Transformer can generalize to other tasks we performed experiments on English\\nconstituency parsing. This task presents specific challenges: the output is subject to strong structural\\nconstraints and is significantly longer than the input. Furthermore, RNN sequence-to-sequence\\nmodels have not been able to attain state-of-the-art results in small-data regimes [37].\\nWe trained a 4-layer transformer with dmodel = 1024 on the Wall Street Journal (WSJ) portion of the\\nPenn Treebank [25], about 40K training sentences. We also trained it in a semi-supervised setting,\\nusing the larger high-confidence and BerkleyParser corpora from with approximately 17M sentences\\n[37]. We used a vocabulary of 16K tokens for the WSJ only setting and a vocabulary of 32K tokens\\nfor the semi-supervised setting.\\nWe performed only a small number of experiments to select the dropout, both attention and residual\\n(section 5.4), learning rates and beam size on the Section 22 development set, all other parameters\\nremained unchanged from the English-to-German base translation model. During inference, we\\n                                                             9\\n---\\nTable 4: The Transformer generalizes well to English constituency parsing (Results are on Section 23\\nof WSJ)\\n                          Parser                           Training             WSJ 23 F1\\n            Vinyals & Kaiser el al. (2014) [37]    WSJ only, discriminative         88.3\\n                 Petrov et al. (2006) [29]         WSJ only, discriminative         90.4\\n                   Zhu et al. (2013) [40]          WSJ only, discriminative         90.4\\n                   Dyer et al. (2016) [8]          WSJ only, discriminative         91.7\\n                  Transformer (4 layers)           WSJ only, discriminative         91.3\\n                   Zhu et al. (2013) [40]              semi-supervised              91.3\\n               Huang & Harper (2009) [14]              semi-supervised              91.3\\n               McClosky et al. (2006) [26]             semi-supervised              92.1\\n            Vinyals & Kaiser el al. (2014) [37]        semi-supervised              92.1\\n                  Transformer (4 layers)               semi-supervised              92.7\\n                 Luong et al. (2015) [23]                  multi-task               93.0\\n                   Dyer et al. (2016) [8]                  generative               93.3\\nincreased the maximum output length to input length + 300. We used a beam size of 21 and α = 0.3\\nfor both WSJ only and the semi-supervised setting.\\nOur results in Table 4 show that despite the lack of task-specific tuning our model performs sur-\\nprisingly well, yielding better results than all previously reported models with the exception of the\\nRecurrent Neural Network Grammar [8].\\nIn contrast to RNN sequence-to-sequence models [37], the Transformer outperforms the Berkeley-\\nParser [29] even when training only on the WSJ training set of 40K sentences.\\n7    Conclusion\\nIn this work, we presented the Transformer, the first sequence transduction model based entirely on\\nattention, replacing the recurrent layers most commonly used in encoder-decoder architectures with\\nmulti-headed self-attention.\\nFor translation tasks, the Transformer can be trained significantly faster than architectures based\\non recurrent or convolutional layers.'"
      ]
     },
-     "execution_count": 12,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -287,8 +287,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -13,12 +13,12 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -45,7 +45,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -55,12 +55,13 @@
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -79,7 +80,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -107,7 +108,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -126,7 +127,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
@@ -157,13 +158,6 @@
   "source": [
    "print(documents[0].text[20000:21000] + \"...\")"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -181,8 +175,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -20,19 +20,11 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "zsh:1: command not found: pip\n"
-     ]
-    }
-   ],
+   "outputs": [],
   "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
   ]
  },
  {
@@ -46,29 +38,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "--2024-03-19 14:30:27--  https://policyholder.gov.in/documents/37343/931203/NBHTGBP22011V012223.pdf/c392bcc1-f6a8-cadd-ab84-495b3273d2c3?version=1.0&t=1669350459879&download=true\n",
-      "Resolving policyholder.gov.in (policyholder.gov.in)... 13.107.246.61, 13.107.213.61\n",
-      "Connecting to policyholder.gov.in (policyholder.gov.in)|13.107.246.61|:443... connected.\n",
-      "HTTP request sent, awaiting response... 200 \n",
-      "Length: 1341586 (1.3M) [application/pdf]\n",
-      "Saving to: ‘./policy.pdf’\n",
-      "\n",
-      "./policy.pdf        100%[===================>]   1.28M  1.37MB/s    in 0.9s    \n",
-      "\n",
-      "2024-03-19 14:30:29 (1.37 MB/s) - ‘./policy.pdf’ saved [1341586/1341586]\n",
-      "\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
   "source": [
    "!wget \"https://policyholder.gov.in/documents/37343/931203/NBHTGBP22011V012223.pdf/c392bcc1-f6a8-cadd-ab84-495b3273d2c3?version=1.0&t=1669350459879&download=true\" -O \"./policy.pdf\""
   ]
@@ -82,34 +54,32 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.openai import OpenAI\n",
@@ -118,7 +88,7 @@
    "from llama_index.core import Settings\n",
    "\n",
    "# for the purpose of this example, we will use the small model embedding and gpt3.5\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm"
@@ -133,16 +103,15 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Started parsing the file under job_id f6ef66a3-a085-4fa5-8300-479adcdea779\n"
+      "Started parsing the file under job_id b8946573-c911-4e00-8921-1bad1cda3d64\n",
+      "......"
     ]
    }
   ],
@@ -154,10 +123,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -193,71 +160,41 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/pierre/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
-      "  warnings.warn(\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Embeddings have been explicitly disabled. Using MockEmbedding.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "111it [00:00, 86184.33it/s]\n",
-      "100%|██████████| 111/111 [00:35<00:00,  3.15it/s]\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
   "source": [
    "nodes = node_parser.get_nodes_from_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "base_nodes, objects = node_parser.get_nodes_and_objects(nodes)\n",
    "\n",
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects)"
+    "recursive_index = VectorStoreIndex(nodes=base_nodes + objects)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = recursive_index.as_query_engine(similarity_top_k=25)"
@@ -272,10 +209,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -301,30 +236,31 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "Started parsing the file under job_id 5b110ac2-e403-4eb3-a92b-97438d97199a\n",
+      "Started parsing the file under job_id ec9e77c9-6ad9-4c9b-9efb-c9f659b0d481\n",
      "....."
     ]
    }
   ],
   "source": [
-    "documents_with_instruction = LlamaParse(result_type=\"markdown\", parsing_instruction=\"\"\"\n",
+    "documents_with_instruction = LlamaParse(\n",
+    "    result_type=\"markdown\",\n",
+    "    parsing_instruction=\"\"\"\n",
    "This document is an insurance policy.\n",
    "When a benefits/coverage/exlusion is describe in the document ammend to it add a text in the follwing benefits string format (where coverage could be an exclusion).\n",
    "\n",
-    "Benefits for {nameofrisk} is {benefitsDescription}, with amount: {benefitsAmount}, and conditions: {benefitsCondition}. \n",
+    "For {nameofrisk} and in this condition {whenDoesThecoverageApply} the coverage is {coverageDescription}. \n",
    "                                        \n",
    "If the document contain a benefits TABLE that describe coverage amounts, do not ouput it as a table, but instead as a list of benefits string.\n",
    "                                       \n",
-    "\"\"\").load_data(\"./policy.pdf\")"
+    "\"\"\",\n",
+    ").load_data(\"./policy.pdf\")"
   ]
  },
  {
@@ -336,158 +272,71 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "## Repatriation of Mortal remains\n",
+      "## Inpatient treatment\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|5K, 10K, 15K, 20K, 25K, 50K, 75K, 100K, 150K, 200K 250K|Nil/50/100|\n",
+      "Claim Form (filled and signed by pe Insured)\n",
+      "Hospital Daily Cash\n",
+      "Release of Medical information Form (filled and signed by pe Insured)\n",
+      "Waiver of Deductible\n",
+      "Original papological and diagnostic reports, discharge summary indoor case papers (if any) and prescriptions issued by pe treating Medical practitioner or Network Provider\n",
+      "Optional Co-payment\n",
+      "Adventure Sports Cover\n",
+      "Home to Home Cover\n",
+      "Passport and Visa copy wip Entry Stamp of Country of Visit and exit Stamp from India\n",
+      "Extension to in-patient care\n",
+      "Ambulance Charge\n",
+      "FIR report of police (if applicable)\n",
      "\n",
-      "## Repatriation of Mortal remains\n",
+      "## Out-patient treatment\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|3Lac, 5Lac|Independent SI|\n",
+      "Cancer Screening & Mammographic Examination\n",
+      "Original bills and receipts for:\n",
+      "1. Charges paid towards Hospital accommodation, nursing facilities, and oper medical services rendered\n",
+      "2. Fees paid to pe Medical Practitioner and for special nursing charges\n",
+      "3. Charges incurred towards any and all test and / or examinations rendered in connection wip pe treatment\n",
+      "4. Charges incurred towards medicines or drugs purchased from a registered pharmacy oper pan pe Network provider duly supported by pe prescriptions of pe Medical Practitioner attending to pe Insured Person\n",
+      "5. Any oper document as required by pe Company to assist pe Claim\n",
      "\n",
-      "## Total Loss of Checked-in Baggage\n",
+      "## Medical evacuation\n",
      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100 to 1000 in multiples of 100|NIL|\n",
+      "Medical reports and transportation details issued by the evacuation agency, prescriptions and medical report by the attending Medical Practitioner furnishing the name of the Insured Person and details of treatment rendered along with the statement confirming the necessity of evacuation.\n",
      "\n",
-      "## Delay of Checked-in Baggage\n",
+      "Documentary proof for expenses incurred towards the Medical Evacuation.\n",
      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100 to 500 in multiples of 100|1/2/3/5/6/24 hours|\n",
-      "|25 to 100 in multiple of 25|3/6/8/12 hours|\n",
+      "## Compassionate visit\n",
      "\n",
-      "## Trip Delay\n",
+      "A certificate from the Medical Practitioner recommending the presence in the form of special assistance to be rendered by an additional member during the entire period of hospitalization. The certificate shall also specify the minimum period in which person is admitted in the hospital.\n",
      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|Post that in multiple of 50 up to 500|hours|\n",
-      "|500, 750, 1000. Post that in|NIL/100/25|\n",
+      "Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
      "\n",
-      "## Trip Cancellation\n",
+      "Stamped boarding pass with invoice used for the travel by the Immediate Family Member.\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|multiple of 1000 till 10000|0/500/100|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|multiple of 500 till 5000|0/500/100|\n",
-      "\n",
-      "## Trip Interruption\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|multiple of 500 till 10000|0/500/100|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|multiple of 500 till 5000|0/500/100|\n",
-      "\n",
-      "## Loss of Passport\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|200 to 500 in multiple of 100|Nil|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100|Independent SI|\n",
-      "\n",
-      "## Loss of International driving License\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|200 to 500 in multiple of 100|Nil|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100|Independent SI|\n",
-      "\n",
-      "## Missed Connection\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|500 to 1000 in multiple of 100|Nil|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100|Independent SI|\n",
-      "\n",
-      "## Financial Emergency Cash\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|500 to 1000 in multiple of 100|NIL|\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|10K, 50K, 1Lac, 2Lac, 3Lac, 4Lac|Nil/100/20|\n",
-      "\n",
-      "## Personal Liability\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|100 % of Base SI|Part of Base SI|\n",
-      "\n",
-      "## Hijack Daily Allowance\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|25, 50, 75, 100 for 7 days|6/12 hours|\n",
-      "\n",
-      "## Bounced Booking-Hotel/Common Carrier\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|100 to 1000 in multiples of 100|6/12 hours|\n",
-      "\n",
-      "|Benefit|USD|\n",
-      "|---|---|\n",
-      "|100|Independent SI|\n",
-      "\n",
-      "## Compassionate Visit\n",
-      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|500 to 5000 in multiple of 500|Nil|\n",
+      "Copy passport of Immediate Family Member with entry and exit stamp.\n",
      "\n",
      "## Escort of Minor Child\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|1000 till 5000 in multiple of 1000|Independent SI|\n",
+      "A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
      "\n",
-      "## Adventure Sport\n",
+      "Discharge summary of the Hospital furnishing details including the date of admission and date of discharge.\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|100 % of Base SI|Part of Base SI|\n",
+      "Stamped Boarding pass used for the return travel of the child to the Country of Residence.\n",
      "\n",
-      "## Sports Equipment Hire\n",
+      "Stamped Boarding pass of the attendant from the Country of Residence to the place of hospitalization (if attendant is necessary).\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|500, 1000, 1500 2000|Nil/100|\n",
+      "Copy of passport of the child with entry and exit stamp.\n",
      "\n",
-      "## Rented sports equipment damage or loss\n",
+      "## Upgradation to Business Class\n",
      "\n",
-      "|Indemnity|USD|\n",
-      "|---|---|\n",
-      "|500, 1000, 1500 2000|Nil/100|\n",
+      "A certificate from the Medical Practitioner specifying the minimum period of Hospitalization.\n",
+      "\n",
+      "Discharge summary of the Hospital furnishing the details including the date of admission and date of discharge.\n",
      "\n",
      "Product Name: Travel infinity | Product UIN: NBHTGBP22011V012223\n",
      "\n",
@@ -497,62 +346,36 @@
      "\n",
      "# Insurance Policy\n",
      "\n",
-      "# Insurance Policy\n",
+      "## Benefits:\n",
      "\n",
-      "Benefits for Repatriation of Mortal remains is Indemnity. Part of Base SI\n",
-      "\n",
-      "Benefits for Repatriation of Mortal remains is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Total Loss of Checked-in Baggage is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Delay of Checked-in Baggage is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Trip Delay is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Trip Cancellation is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Trip Cancellation is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Trip Interruption is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Trip Interruption is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Loss of Passport is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Loss of Passport is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Loss of International driving License is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Loss of International driving License is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Missed Connection is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Missed Connection is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Financial Emergency Cash is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Personal Liability is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Hijack Daily Allowance is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Bounced Booking-Hotel/Common Carrier is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Bounced Booking-Hotel/Common Carrier is Benefit. Independent SI\n",
-      "\n",
-      "Benefits for Compassionate Visit is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Escort of Minor Child is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Adventure Sport is Indemnity. Part of Base SI\n",
-      "\n",
-      "Benefits for Sports Equipment Hire is Indemnity. Independent SI\n",
-      "\n",
-      "Benefits for Rented sports equipment damage or loss is Indemnity. Independent SI\n"
+      "- For Inpatient treatment and in this condition when admitted to a hospital, the coverage is reimbursement for medical expenses incurred.\n",
+      "- For Hospital Daily Cash and in this condition when hospitalized, the coverage is daily cash benefit.\n",
+      "- For Waiver of Deductible and in this condition when a deductible is applicable, the coverage is waiver of the deductible amount.\n",
+      "- For Optional Co-payment and in this condition when a co-payment is required, the coverage is optional co-payment.\n",
+      "- For Adventure Sports Cover and in this condition when participating in adventure sports, the coverage is coverage for injuries related to adventure sports.\n",
+      "- For Home to Home Cover and in this condition when requiring medical evacuation, the coverage is assistance for repatriation to home country.\n",
+      "- For Extension to in-patient care and in this condition when extended hospital stay is necessary, the coverage is extension of coverage for in-patient care.\n",
+      "- For Ambulance Charge and in this condition when ambulance services are utilized, the coverage is reimbursement for ambulance charges.\n",
+      "- For Out-patient treatment and in this condition when receiving outpatient medical care, the coverage is reimbursement for outpatient medical expenses.\n",
+      "- For Cancer Screening & Mammographic Examination and in this condition when undergoing cancer screening or mammographic examination, the coverage is coverage for these preventive services.\n",
+      "- For New Born baby Cover and in this condition when a newborn is covered under the policy, the coverage is medical expenses coverage for the newborn.\n",
+      "- For Maternity and in this condition when maternity services are required, the coverage is coverage for maternity expenses.\n",
+      "- For Complete pre-existing disease cover and in this condition when seeking treatment for pre-existing conditions, the coverage is coverage for pre-existing conditions.\n",
+      "- For Medical sum insured replenishment in case of hospitalization due to accident and in this condition when hospitalized due to an accident, the coverage is replenishment of the sum insured.\n",
+      "- For Waiver of sublimit for insured above 60 years of age and in this condition when the insured is above 60 years of age, the coverage is waiver of sublimits.\n",
+      "- For Psychiatric Counseling and in this condition when seeking psychiatric counseling, the coverage is coverage for psychiatric counseling services.\n",
+      "- For Physiotherapy and in this condition when undergoing physiotherapy, the coverage is coverage for physiotherapy sessions.\n",
+      "- For Terrorism cover and in this condition when affected by terrorism, the coverage is coverage for medical expenses related to terrorism incidents.\n",
+      "- For Medical tele-consultation and in this condition when consulting a medical practitioner remotely, the coverage is coverage for tele-consultation services.\n",
+      "- For Medical evacuation and in this condition when requiring medical evacuation, the coverage is coverage for medical evacuation services.\n",
+      "- For Compassionate visit and in this condition when requiring a compassionate visit, the coverage is coverage for travel expenses for a family member to visit.\n",
+      "- For Escort of Minor Child and in this condition when escorting a minor child for medical treatment, the coverage is coverage for escort services for the child.\n",
+      "- For Upgradation to Business Class and in this condition when requiring upgradation to business class for medical travel, the coverage is coverage for upgradation to business class.\n"
     ]
    }
   ],
   "source": [
-    "target_page = 51\n",
+    "target_page = 45\n",
    "pages_vanilla = documents[0].text.split(\"\\n---\\n\")\n",
    "pages_with_instructions = documents_with_instruction[0].text.split(\"\\n---\\n\")\n",
    "\n",
@@ -563,34 +386,25 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Embeddings have been explicitly disabled. Using MockEmbedding.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "1it [00:00, 25575.02it/s]\n",
-      "100%|██████████| 1/1 [00:02<00:00,  2.13s/it]\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
   "source": [
-    "node_parser_instruction = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)\n",
+    "node_parser_instruction = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")\n",
    "nodes_instruction = node_parser.get_nodes_from_documents(documents_with_instruction)\n",
-    "base_nodes_instruction, objects_instruction = node_parser_instruction.get_nodes_and_objects(nodes_instruction)\n",
+    "(\n",
+    "    base_nodes_instruction,\n",
+    "    objects_instruction,\n",
+    ") = node_parser_instruction.get_nodes_and_objects(nodes_instruction)\n",
    "\n",
-    "recursive_index_instruction = VectorStoreIndex(nodes=base_nodes_instruction+objects_instruction)\n",
-    "query_engine_instruction = recursive_index_instruction.as_query_engine(similarity_top_k=25)"
+    "recursive_index_instruction = VectorStoreIndex(\n",
+    "    nodes=base_nodes_instruction + objects_instruction\n",
+    ")\n",
+    "query_engine_instruction = recursive_index_instruction.as_query_engine(\n",
+    "    similarity_top_k=25\n",
+    ")"
   ]
  },
  {
@@ -604,19 +418,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vanilla:\n",
-      "You are covered for the expenses you incurred due to the delay of your trip. The amount you paid, which was 45, will be covered up to the limit specified in the certificate of insurance for every block of hours of delay, as mentioned in the policy.\n",
+      "You are covered for the amount you paid due to the trip delay, up to the limit specified in the certificate of insurance.\n",
      "With instructions:\n",
-      "For Trip Delay coverage, the payment amount for every block of hours of delay is as mentioned in the certificate of insurance.\n"
+      "For Trip Delay coverage, you are covered for a fixed benefit amount as mentioned in the certificate of insurance for every block of hours of delay.\n"
     ]
    }
   ],
@@ -629,7 +441,7 @@
    "\n",
    "print(\"With instructions:\")\n",
    "response_1_i = query_engine_instruction.query(query_1)\n",
-    "print(response_1_i)\n"
+    "print(response_1_i)"
   ]
  },
  {
@@ -641,10 +453,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -671,10 +481,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 18,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -698,20 +506,13 @@
    "response_3_i = query_engine_instruction.query(query_3)\n",
    "print(response_3_i)"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "llama_parse",
   "language": "python",
-   "name": "python3"
+   "name": "llama_parse"
  },
  "language_info": {
   "codemirror_mode": {
@@ -722,8 +523,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.6"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -40,18 +40,18 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "id": "0879301c-ff91-4431-941a-6c0ef7cd8fe2",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -63,9 +63,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "391e2d95-5569-4d73-9f16-5b59d7326f8d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.llms.anthropic import Anthropic\n",
@@ -75,11 +73,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "id": "700f48e8-8b52-41f3-90f9-144d5fdd5c52",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import Settings\n",
@@ -102,12 +98,10 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "c39d408f-e885-4940-85c7-b09ca3bc7cb7",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -122,11 +116,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "9c9cd670-8229-4ad6-99a9-845bd82b7ec1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -146,11 +138,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "b26d21d1-05b5-4f49-b937-c13106a84015",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.schema import TextNode\n",
@@ -160,23 +150,16 @@
    "def get_text_nodes(json_list: List[dict]):\n",
    "    text_nodes = []\n",
    "    for idx, page in enumerate(json_list):\n",
-    "        text_node = TextNode(\n",
-    "            text=page[\"text\"],\n",
-    "            metadata={\n",
-    "                \"page\": page[\"page\"]\n",
-    "            }\n",
-    "        )\n",
+    "        text_node = TextNode(text=page[\"text\"], metadata={\"page\": page[\"page\"]})\n",
    "        text_nodes.append(text_node)\n",
    "    return text_nodes"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "id": "364a3276-d2db-4aee-9bc6-617ffd726d25",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "text_nodes = get_text_nodes(json_list)"
@@ -194,7 +177,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "36012145-5521-4ddb-a53e-df9ebd1ca8dd",
   "metadata": {},
   "outputs": [
@@ -223,13 +206,10 @@
    "    for image_dict in image_dicts:\n",
    "        image_doc = ImageDocument(image_path=image_dict[\"path\"])\n",
    "        response = anthropic_mm_llm.complete(\n",
-    "            prompt=\"Describe the images as an alternative text\",\n",
+    "            prompt=\"Describe the images as alt text\",\n",
    "            image_documents=[image_doc],\n",
    "        )\n",
-    "        text_node = TextNode(\n",
-    "            text=str(response),\n",
-    "            metadata={\"path\": image_dict[\"path\"]}\n",
-    "        )\n",
+    "        text_node = TextNode(text=str(response), metadata={\"path\": image_dict[\"path\"]})\n",
    "        img_text_nodes.append(text_node)\n",
    "    return img_text_nodes"
   ]
@@ -238,9 +218,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "38f25045-6102-4920-9cd0-42b0ae6c872f",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "image_text_nodes = get_image_text_nodes(json_objs)"
@@ -248,11 +226,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 60,
+   "execution_count": null,
   "id": "4683c97a-da06-408a-9fe9-7e3c0aceb77d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "data": {
@@ -260,7 +236,7 @@
       "'The image shows a bar graph titled \"Monthly Active Platform Consumers (in millions)\". The graph displays data from Q2 2020 to Q1 2022 over 8 quarters. The number of monthly active platform consumers starts at 55 million in Q2 2020 and steadily increases each quarter, reaching 115 million by Q1 2022. The graph illustrates consistent quarter-over-quarter growth in this metric over the nearly 2 year time period shown.'"
      ]
     },
-     "execution_count": 60,
+     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
@@ -281,7 +257,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 68,
+   "execution_count": null,
   "id": "939aec6c-064a-4319-b2dc-70cc4a304c06",
   "metadata": {},
   "outputs": [],
@@ -293,7 +269,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 69,
+   "execution_count": null,
   "id": "529340d5-9319-4cdf-8ee1-bbd01ed00226",
   "metadata": {},
   "outputs": [],
@@ -303,11 +279,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 71,
+   "execution_count": null,
   "id": "81d7ff30-5a87-44da-880d-4b1f41434d90",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -320,18 +294,18 @@
    }
   ],
   "source": [
-    "# ask question over image! \n",
-    "response = query_engine.query(\"What does the bar graph titled 'Monthly Active Platform Consumers' show?\") \n",
-    "print(str(response)) "
+    "# ask question over image!\n",
+    "response = query_engine.query(\n",
+    "    \"What does the bar graph titled 'Monthly Active Platform Consumers' show?\"\n",
+    ")\n",
+    "print(str(response))"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 72,
+   "execution_count": null,
   "id": "c4f14ad8-6bfd-49d9-b3d5-7215cf0e4ac1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -360,25 +334,17 @@
    }
   ],
   "source": [
-    "# ask question over text! \n",
-    "response = query_engine.query(\"What are the main risk factors for Uber?\") \n",
-    "print(str(response)) "
+    "# ask question over text!\n",
+    "response = query_engine.query(\"What are the main risk factors for Uber?\")\n",
+    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "82ea880b-a0c7-410c-94c7-8fb3ac96c30c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
-   "display_name": "llama_parse",
+   "display_name": "llama-parse-aNC435Vv-py3.10",
   "language": "python",
-   "name": "llama_parse"
+   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
@@ -389,8 +355,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -33,27 +33,27 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-llms-anthropic llama-index-multi-modal-llms-anthropic\n",
-    "!pip install llama-index-embeddings-huggingface\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-llms-anthropic llama-index-multi-modal-llms-anthropic\n",
+    "%pip install llama-index-embeddings-huggingface\n",
+    "%pip install llama-parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "0879301c-ff91-4431-941a-6c0ef7cd8fe2",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -73,12 +73,10 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "391e2d95-5569-4d73-9f16-5b59d7326f8d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/10q/uber_10q_march_2022.pdf' -O './uber_10q_march_2022.pdf'"
   ]
  },
  {
@@ -91,7 +89,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "id": "33f2a06e",
   "metadata": {},
   "outputs": [
@@ -129,7 +127,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
   "id": "e1e9e955",
   "metadata": {},
   "outputs": [
@@ -157,7 +155,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "3108ba82",
   "metadata": {},
   "outputs": [
@@ -188,7 +186,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "id": "07669c6a",
   "metadata": {},
   "outputs": [
@@ -220,10 +218,12 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "from llama_index.core.node_parser import LlamaParseJsonNodeParser \n",
+    "from llama_index.core.node_parser import LlamaParseJsonNodeParser\n",
    "from llama_index.llms.openai import OpenAI\n",
    "\n",
-    "node_parser = LlamaParseJsonNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo\"), num_workers=16, include_metadata=True)\n",
+    "node_parser = LlamaParseJsonNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo\"), num_workers=16, include_metadata=True\n",
+    ")\n",
    "nodes = node_parser.get_nodes_from_documents(documents)\n",
    "base_nodes, objects = node_parser.get_nodes_and_objects(nodes)"
   ]
@@ -238,7 +238,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
   "id": "71b75550",
   "metadata": {},
   "outputs": [
@@ -260,17 +260,18 @@
    "from llama_index.core import VectorStoreIndex\n",
    "\n",
    "\n",
-    "recursive_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "recursive_index = VectorStoreIndex(nodes=base_nodes + objects)\n",
    "recursive_query_engine = recursive_index.as_query_engine(\n",
-    "    similarity_top_k=5, \n",
-    "    verbose=True\n",
+    "    similarity_top_k=5, verbose=True\n",
    ")\n",
-    "res = recursive_query_engine.query(\"what is UBER Short-term insurance reserves reported in 2022\")\n"
+    "res = recursive_query_engine.query(\n",
+    "    \"what is UBER Short-term insurance reserves reported in 2022\"\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
   "id": "965fdd22",
   "metadata": {},
   "outputs": [
@@ -301,7 +302,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": null,
   "id": "0f917e20",
   "metadata": {},
   "outputs": [
@@ -322,12 +323,14 @@
    }
   ],
   "source": [
-    "res = recursive_query_engine.query(\"what is Comprehensive income (loss) attributable to Uber reported in 2021\")\n"
+    "res = recursive_query_engine.query(\n",
+    "    \"what is Comprehensive income (loss) attributable to Uber reported in 2021\"\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
   "id": "85b5c025",
   "metadata": {},
   "outputs": [
@@ -372,8 +375,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -21,16 +21,14 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "87322210-c21c-43d6-b459-2e8a828ac576",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
@@ -39,6 +37,7 @@
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\""
   ]
  },
@@ -56,9 +55,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "e81e0a08-3a99-42e6-adcc-00bb4ce1c3d4",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "!wget \"https://www.dropbox.com/scl/fi/fxg17log5ydwoflhxmgrb/treasury_report.pdf?rlkey=mdintk0o2uuzkple26vc4v6fd&dl=1\" -O treasury_report.pdf"
@@ -66,11 +63,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "ecfc578c-3c7f-4ec1-aa06-51565c28632b",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -84,20 +79,15 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"fr\"\n",
-    ")\n",
+    "parser = LlamaParse(result_type=\"text\", language=\"fr\")\n",
    "documents = parser.load_data(\"./treasury_report.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "0c37db27-3496-4a59-918b-701c9ad7706d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -209,11 +199,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
   "id": "ac332ea3-cfff-4216-b292-62410a26c336",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -248,11 +236,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
   "id": "45235b17-08f0-48f1-92aa-06711225860b",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -266,20 +252,15 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"ch_sim\"\n",
-    ")\n",
+    "parser = LlamaParse(result_type=\"text\", language=\"ch_sim\")\n",
    "documents = parser.load_data(\"./chinese_pdf.pdf\")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
   "id": "f0d546cc-6549-4cf5-8b37-0896f4e8d43d",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -401,9 +382,7 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "640f0679-7f7e-4b0a-a46d-b099ae382fe2",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "# download another copy with a different name to avoid hitting pdf cache\n",
@@ -412,11 +391,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": null,
   "id": "bfcacf90-ca67-4bfd-b023-be0af2cb18c5",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -429,10 +406,7 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "base_parser = LlamaParse(\n",
-    "    result_type=\"text\",\n",
-    "    language=\"en\"\n",
-    ")\n",
+    "base_parser = LlamaParse(result_type=\"text\", language=\"en\")\n",
    "base_documents = parser.load_data(\"./chinese_pdf2.pdf\")"
   ]
  },
@@ -440,21 +414,11 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "b264ed4e-647a-4f51-9f79-fdf82b76762a",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "print(base_documents[0].get_content()[1000:10000])"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d9f02762-bb97-4e0e-8268-ccc00612a974",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -472,8 +436,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -0,0 +1,368 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# LlamaParse With MongoDB\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_mongodb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
+    "\n",
+    "In this notebook, we provide a straightforward example of using LlamaParse with MongoDB Atlas VectorSearch.\n",
+    "\n",
+    "We illustrate the process of using llama-parse to parse a PDF document, then index the document with a MongoDB vector store, and subsequently perform basic queries against this store.\n",
+    "\n",
+    "This notebook is structured similarly to quick start guides, aiming to introduce users to utilizing llama-parse in conjunction with a MongoDB Atlas VectorSearch."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Installation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install llama-index llama-parse\n",
+    "%pip install llama-index-vector-stores-mongodb llama-index-llms-openai"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Setup API Keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\n",
+    "    \"LLAMA_CLOUD_API_KEY\"\n",
+    "] = \"\"  # Get it from https://cloud.llamaindex.ai/api-key\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"\"  # Get it from https://platform.openai.com/api-keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# llama-parse is async-first, running the sync code in a notebook requires the use of nest_asyncio\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "import requests\n",
+    "import pymongo\n",
+    "\n",
+    "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
+    "from llama_parse import LlamaParse\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "from llama_index.core import VectorStoreIndex, StorageContext\n",
+    "from llama_index.core.node_parser import SimpleNodeParser"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Download Document\n",
+    "\n",
+    "We will use `Attention is all you need` paper."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Download complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# The URL of the file you want to download\n",
+    "url = \"https://arxiv.org/pdf/1706.03762.pdf\"\n",
+    "# The local path where you want to save the file\n",
+    "file_path = \"./attention.pdf\"\n",
+    "\n",
+    "# Perform the HTTP request\n",
+    "response = requests.get(url)\n",
+    "\n",
+    "# Check if the request was successful\n",
+    "if response.status_code == 200:\n",
+    "    # Open the file in binary write mode and save the content\n",
+    "    with open(file_path, \"wb\") as file:\n",
+    "        file.write(response.content)\n",
+    "    print(\"Download complete.\")\n",
+    "else:\n",
+    "    print(\"Error downloading the file.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Parse the document using `LlamaParse`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Started parsing the file under job_id 09a49745-9f21-4190-9de8-27e4e1a4bdf5\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = LlamaParse(result_type=\"text\").load_data(file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "rmer - model architecture.\n",
+      "The Transformer follows this overall architecture using stacked self-attention and point-wise, fully\n",
+      "connected layers for both the encoder and decoder, shown in the left and right halves of Figure 1,\n",
+      "respectively.\n",
+      "3.1   Encoder and Decoder Stacks\n",
+      "Encoder:     The encoder is composed of a stack of N = 6 identical layers. Each layer has two\n",
+      "sub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-\n",
+      "wise fully connected feed-forward network. We employ a residual connection [11] around each of\n",
+      "the two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is\n",
+      "LayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer\n",
+      "itself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding\n",
+      "layers, produce outputs of dimension dmodel = 512.\n",
+      "Decoder:    The decoder is also composed of a stack of N = 6 identical layers. In addition \n"
+     ]
+    }
+   ],
+   "source": [
+    "# Take a quick look at some of the parsed text from the document:\n",
+    "print(documents[0].get_content()[10000:11000])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create `MongoDBAtlasVectorSearch`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mongo_uri = os.environ[\"MONGO_URI\"]\n",
+    "\n",
+    "mongodb_client = pymongo.MongoClient(mongo_uri)\n",
+    "mongodb_vector_store = MongoDBAtlasVectorSearch(mongodb_client)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create nodes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "node_parser = SimpleNodeParser()\n",
+    "\n",
+    "nodes = node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create Index and Query Engine."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(vector_store=mongodb_vector_store)\n",
+    "\n",
+    "index = VectorStoreIndex(\n",
+    "    nodes=nodes,\n",
+    "    storage_context=storage_context,\n",
+    "    embed_model=OpenAIEmbedding(),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_engine = index.as_query_engine(similarity_top_k=2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Test Query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "***********New LlamaParse+ Basic Query Engine***********\n",
+      "The BLEU score on the WMT 2014 English-to-German translation task is 28.4.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query = \"What is BLEU score on the WMT 2014 English-to-German translation task?\"\n",
+    "\n",
+    "response = query_engine.query(query)\n",
+    "print(\"\\n***********New LlamaParse+ Basic Query Engine***********\")\n",
+    "print(response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "We varied the learning\n",
+      "rate over the course of training, according to the formula:\n",
+      "               lrate = d−0.5                                                                          (3)\n",
+      "                         model · min(step_num−0.5, step_num · warmup_steps−1.5)\n",
+      "This corresponds to increasing the learning rate linearly for the first warmup_steps training steps,\n",
+      "and decreasing it thereafter proportionally to the inverse square root of the step number. We used\n",
+      "warmup_steps = 4000.\n",
+      "5.4   Regularization\n",
+      "We employ three types of regularization during training:\n",
+      "                                                    7\n",
+      "---\n",
+      "Table 2: The Transformer achieves better BLEU scores than previous state-of-the-art models on the\n",
+      "English-to-German and English-to-French newstest2014 tests at a fraction of the training cost.\n",
+      "        Model                                          BLEU               Training Cost (FLOPs)\n",
+      "                                                 EN-DE      EN-FR          EN-DE         EN-FR\n",
+      "        ByteNet [18]                              23.75\n",
+      "        Deep-Att + PosUnk [39]                               39.2                       1.0 · 1020\n",
+      "        GNMT + RL [38]                            24.6       39.92        2.3 · 1019    1.4 · 1020\n",
+      "        ConvS2S [9]                               25.16      40.46        9.6 · 1018    1.5 · 1020\n",
+      "        MoE [32]                                  26.03      40.56        2.0 · 1019    1.2 · 1020\n",
+      "        Deep-Att + PosUnk Ensemble [39]                      40.4                       8.0 · 1020\n",
+      "        GNMT + RL Ensemble [38]                   26.30      41.16        1.8 · 1020    1.1 · 1021\n",
+      "        ConvS2S Ensemble [9]                      26.36      41.29        7.7 · 1019    1.2 · 1021\n",
+      "        Transformer (base model)                  27.3       38.1               3.3 · 1018\n",
+      "        Transformer (big)                         28.4       41.8                2.3 · 1019\n",
+      "Residual Dropout       We apply dropout [33] to the output of each sub-layer, before it is added to the\n",
+      "sub-layer input and normalized. In addition, we apply dropout to the sums of the embeddings and the\n",
+      "positional encodings in both the encoder and decoder stacks. For the base model, we use a rate of\n",
+      "Pdrop = 0.1.\n",
+      "Label Smoothing        During training, we employed label smoothing of value ϵls = 0.1 [36]. This\n",
+      "hurts perplexity, as the model learns to be more unsure, but improves accuracy and BLEU score.\n",
+      "6    Results\n",
+      "6.1   Machine Translation\n",
+      "On the WMT 2014 English-to-German translation task, the big transformer model (Transformer (big)\n",
+      "in Table 2) outperforms the best previously reported models (including ensembles) by more than 2.0\n",
+      "BLEU, establishing a new state-of-the-art BLEU score of 28.4. The configuration of this model is\n",
+      "listed in the bottom line of Table 3. Training took 3.5 days on 8 P100 GPUs. Even our base model\n",
+      "surpasses all previously published models and ensembles, at a fraction of the training cost of any of\n",
+      "the competitive models.\n",
+      "On the WMT 2014 English-to-French translation task, our big model achieves a BLEU score of 41.0,\n",
+      "outperforming all of the previously published single models, at less than 1/4 the training cost of the\n",
+      "previous state-of-the-art model. The Transformer (big) model trained for English-to-French used\n",
+      "dropout rate Pdrop = 0.1, instead of 0.3.\n",
+      "For the base models, we used a single model obtained by averaging the last 5 checkpoints, which\n",
+      "were written at 10-minute intervals. For the big models, we averaged the last 20 checkpoints. We\n",
+      "used beam search with a beam size of 4 and length penalty α = 0.6 [38]. These hyperparameters\n",
+      "were chosen after experimentation on the development set. We set the maximum output length during\n",
+      "inference to input length + 50, but terminate early when possible [38].\n",
+      "Table 2 summarizes our results and compares our translation quality and training costs to other model\n",
+      "architectures from the literature.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Take a look at one of the source nodes from the response\n",
+    "print(response.source_nodes[0].get_content())"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "anthropic_env",
+   "language": "python",
+   "name": "anthropic_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
@@ -28,13 +28,13 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-question-gen-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-question-gen-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
   ]
  },
  {
@@ -56,15 +56,17 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama-parse is async-first, running the async code in a notebook requires the use of nest_asyncio\n",
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "import os\n",
+    "\n",
    "# API access to llama-cloud\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\"\n",
    "\n",
@@ -74,7 +76,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
@@ -83,11 +85,11 @@
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core import Settings\n",
    "\n",
-    "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
+    "embed_model = OpenAIEmbedding(model=\"text-embedding-3-small\")\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo-0125\")\n",
    "\n",
    "Settings.llm = llm\n",
-    "Settings.embed_model = embed_model\n"
+    "Settings.embed_model = embed_model"
   ]
  },
  {
@@ -109,8 +111,8 @@
   "source": [
    "from llama_parse import LlamaParse\n",
    "\n",
-    "docs_2021 = LlamaParse(result_type=\"markdown\").load_data('./apple_2021_10k.pdf')\n",
-    "docs_2020 = LlamaParse(result_type=\"markdown\").load_data('./apple_2020_10k.pdf')"
+    "docs_2021 = LlamaParse(result_type=\"markdown\").load_data(\"./apple_2021_10k.pdf\")\n",
+    "docs_2020 = LlamaParse(result_type=\"markdown\").load_data(\"./apple_2020_10k.pdf\")"
   ]
  },
  {
@@ -127,31 +129,34 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.node_parser import MarkdownElementNodeParser\n",
    "\n",
-    "node_parser = MarkdownElementNodeParser(llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8)"
+    "node_parser = MarkdownElementNodeParser(\n",
+    "    llm=OpenAI(model=\"gpt-3.5-turbo-0125\"), num_workers=8\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
-    "from llama_index.postprocessor.flag_embedding_reranker import FlagEmbeddingReranker\n",
+    "from llama_index.postprocessor.flag_embedding_reranker import (\n",
+    "    FlagEmbeddingReranker,\n",
+    ")\n",
    "\n",
    "reranker = FlagEmbeddingReranker(\n",
    "    top_n=5,\n",
    "    model=\"BAAI/bge-reranker-large\",\n",
    ")\n",
    "\n",
+    "\n",
    "def create_query_engine_over_doc(docs, nodes_save_path=None):\n",
    "    \"\"\"Big function to go from document path -> recursive retriever.\"\"\"\n",
    "    if nodes_save_path is not None and os.path.exists(nodes_save_path):\n",
@@ -161,16 +166,13 @@
    "        if nodes_save_path is not None:\n",
    "            pickle.dump(raw_nodes, open(nodes_save_path, \"wb\"))\n",
    "\n",
-    "    base_nodes, objects = node_parser.get_nodes_and_objects(\n",
-    "        raw_nodes\n",
-    "    )\n",
+    "    base_nodes, objects = node_parser.get_nodes_and_objects(raw_nodes)\n",
    "\n",
    "    ### Construct Retrievers\n",
    "    # construct top-level vector index + query engine\n",
-    "    vector_index = VectorStoreIndex(nodes=base_nodes+objects)\n",
+    "    vector_index = VectorStoreIndex(nodes=base_nodes + objects)\n",
    "    query_engine = vector_index.as_query_engine(\n",
-    "        similarity_top_k=15,\n",
-    "        node_postprocessors=[reranker]\n",
+    "        similarity_top_k=15, node_postprocessors=[reranker]\n",
    "    )\n",
    "    return query_engine, base_nodes"
   ]
@@ -178,9 +180,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine_2021, nodes_2021 = create_query_engine_over_doc(\n",
@@ -193,10 +193,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 38,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
@@ -209,18 +207,14 @@
    "        query_engine=query_engine_2021,\n",
    "        metadata=ToolMetadata(\n",
    "            name=\"apple_2021_10k\",\n",
-    "            description=(\n",
-    "                \"Provides information about Apple financials for year 2021\"\n",
-    "            ),\n",
+    "            description=(\"Provides information about Apple financials for year 2021\"),\n",
    "        ),\n",
    "    ),\n",
    "    QueryEngineTool(\n",
    "        query_engine=query_engine_2020,\n",
    "        metadata=ToolMetadata(\n",
    "            name=\"apple_2020_10k\",\n",
-    "            description=(\n",
-    "                \"Provides information about Apple financials for year 2020\"\n",
-    "            ),\n",
+    "            description=(\"Provides information about Apple financials for year 2020\"),\n",
    "        ),\n",
    "    ),\n",
    "]\n",
@@ -241,10 +235,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 41,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -271,10 +263,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 40,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -290,10 +280,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 42,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -316,10 +304,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -342,10 +328,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 45,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -358,13 +342,6 @@
   "source": [
    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -382,8 +359,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -18,12 +18,11 @@
   "cell_type": "code",
   "execution_count": null,
   "id": "14cdcfaf-88b4-4489-9910-e362e0ccec53",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
+    "\n",
    "nest_asyncio.apply()\n",
    "\n",
    "from llama_parse import LlamaParse"
@@ -31,12 +30,13 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "6f5b5841-dd3e-4169-9bd4-6a672b5b34ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-\""
   ]
  },
@@ -68,7 +68,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
   "id": "0dd0f860-8e92-43a7-9443-ad1a4fb9365c",
   "metadata": {},
   "outputs": [],
@@ -78,7 +78,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
   "id": "fd932bef-ba82-4449-b7a0-5c2a9b55089f",
   "metadata": {},
   "outputs": [
@@ -104,7 +104,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
   "id": "2a73e553-2194-4ac9-9764-0edab0d6fdce",
   "metadata": {},
   "outputs": [
@@ -308,11 +308,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
   "id": "c779547f-e4f7-4c84-9786-2b6b749827ab",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import VectorStoreIndex"
@@ -320,7 +318,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
   "id": "68b3a95e-ce19-4df1-9fdd-e6efb2fc423a",
   "metadata": {},
   "outputs": [],
@@ -330,11 +328,9 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
   "id": "a2ae28f6-4b3a-4130-8e65-0921b7678739",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = index.as_query_engine()"
@@ -342,23 +338,21 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
   "id": "232091ee-aa22-4f51-838c-410024acc344",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "response = query_engine.query(\"What are some response quality challenges with naive RAG?\") "
+    "response = query_engine.query(\n",
+    "    \"What are some response quality challenges with naive RAG?\"\n",
+    ")"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
   "id": "75f32aa7-c308-4221-af60-779822cfdba1",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -371,14 +365,6 @@
   "source": [
    "print(str(response))"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d309d8fb-750a-4393-a1b2-67b14b7c121f",
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -396,8 +382,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -2,9 +2,7 @@
 "cells": [
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "eld1dKaN7P8B"
-   },
+   "metadata": {},
   "source": [
    "# LlamaParse - Parsing Financial Powerpoints 📊\n",
    "\n",
@@ -13,9 +11,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "goB1sV8zu_Xl"
-   },
+   "metadata": {},
   "source": [
    "## Installation\n",
    "\n",
@@ -27,25 +23,17 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "7Y3_BwQLu-qK",
-    "outputId": "b1129c52-7a70-44cc-ad03-1f8d3a8c794a"
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-parse\n",
-    "!pip install torch transformers python-pptx Pillow"
+    "%pip install llama-index\n",
+    "%pip install llama-parse\n",
+    "%pip install torch transformers python-pptx Pillow"
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "i-Rg2D_Rvf2i"
-   },
+   "metadata": {},
   "source": [
    "## API Key\n",
    "\n",
@@ -55,32 +43,26 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "id": "af6i2P1vuU-U"
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
+    "\n",
    "os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-...\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "p8Eq-aX-wAEo"
-   },
+   "metadata": {},
   "source": [
    "**NOTE**: Since LlamaParse is natively async, running the sync code in a notebook requires the use of nest_asyncio.\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "id": "4OB0BkTqv_0l",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
@@ -90,9 +72,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "dz927ecMyYo_"
-   },
+   "metadata": {},
   "source": [
    "## Importing the package\n",
    "\n",
@@ -102,10 +82,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "id": "nSW-6sEwyXwx",
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_parse import LlamaParse"
@@ -113,9 +90,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "l_D4YsAHwUSk"
-   },
+   "metadata": {},
   "source": [
    "## Using LlamaParse to Parse Presentations\n",
    "\n",
@@ -127,14 +102,7 @@
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "d3qeuiyawT0U",
-    "outputId": "cec0ea0a-be8b-49b6-9376-797c91f63be7",
-    "tags": []
-   },
+   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir data; wget \"https://meetings.wmo.int/Cg-19/PublishingImages/SitePages/FINAC-43/7%20-%20EC-77-Doc%205%20Financial%20Statements%20for%202022%20(FINAC).pptx\" -O data/presentation.pptx"
@@ -142,9 +110,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "Gbr8RiHEyF3-"
-   },
+   "metadata": {},
   "source": [
    "### Parsing the presentation\n",
    "\n",
@@ -155,24 +121,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "osocsofoJ42S"
-   },
+   "metadata": {},
   "source": [
    "#### Llama Index default"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "PTVy5XCNJwW-",
-    "outputId": "d0e2cc4b-1407-45a9-b5e6-d06f91a533b4",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import SimpleDirectoryReader\n",
@@ -182,24 +139,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "oucbsciZJwxt"
-   },
+   "metadata": {},
   "source": [
    "#### Llama Parse"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "3jKnXCuAyQ9_",
-    "outputId": "1f668f17-1e20-46e5-fbab-9a55e4b28891",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -210,7 +158,9 @@
    }
   ],
   "source": [
-    "llama_parse_documents = LlamaParse(result_type=\"markdown\").load_data(\"./data/presentation.pptx\")"
+    "llama_parse_documents = LlamaParse(result_type=\"markdown\").load_data(\n",
+    "    \"./data/presentation.pptx\"\n",
+    ")"
   ]
  },
  {
@@ -224,10 +174,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -259,9 +207,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
   "source": [
    "Compared against the original slide image.\n",
    "![Demo](demo_ppt_financial_1.png)"
@@ -269,9 +215,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "p4GVOdWzzvYg"
-   },
+   "metadata": {},
   "source": [
    "## Comparing the two for RAG\n",
    "\n",
@@ -280,20 +224,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "oVcdGus5NDxi"
-   },
+   "metadata": {},
   "source": [
    "### Query Engine on SimpleDirectoryReader results"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {
-    "id": "DqXYsLCWNg9_",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
@@ -304,20 +243,15 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "ZLkHt9l2Nbxx"
-   },
+   "metadata": {},
   "source": [
    "### Query Engine on LlamaParse Results\n"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {
-    "id": "ZllaDcfRNLv3",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [],
   "source": [
    "llama_parse_index = VectorStoreIndex.from_documents(llama_parse_documents)\n",
@@ -326,10 +260,7 @@
  },
  {
   "cell_type": "markdown",
-   "metadata": {
-    "id": "0dY_0_1bNg0X",
-    "tags": []
-   },
+   "metadata": {},
   "source": [
    "### Liability provision\n",
    "What was the liability provision as of Dec 31 2021?\n",
@@ -339,15 +270,8 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "Tmn-qNTEN-cb",
-    "outputId": "a9bffc00-9cfc-43d8-b159-596a6c1aca64",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -358,21 +282,16 @@
    }
   ],
   "source": [
-    "vanilla_response = vanilla_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
+    "vanilla_response = vanilla_query_engine.query(\n",
+    "    \"What was the liability provision as of Dec 31 2021?\"\n",
+    ")\n",
    "print(vanilla_response)"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "4EZ_uqlROP7R",
-    "outputId": "0645a159-06c6-411e-d1f6-79ea95d32b42",
-    "tags": []
-   },
+   "execution_count": null,
+   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
@@ -383,16 +302,11 @@
    }
   ],
   "source": [
-    "llama_parse_response = llama_parse_query_engine.query(\"What was the liability provision as of Dec 31 2021?\")\n",
+    "llama_parse_response = llama_parse_query_engine.query(\n",
+    "    \"What was the liability provision as of Dec 31 2021?\"\n",
+    ")\n",
    "print(llama_parse_response)"
   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
  }
 ],
 "metadata": {
@@ -413,8 +327,7 @@
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
@@ -1,3 +1,3 @@
 from llama_parse.base import LlamaParse, ResultType

-__all__ = ["LlamaParse", "ResultType"]
+__all__ = ["LlamaParse", "ResultType"]
@@ -3,7 +3,6 @@ import asyncio
 import httpx
 import mimetypes
 import time
-from enum import Enum
 from pathlib import Path
 from typing import List, Optional, Union

@@ -12,113 +11,13 @@ from llama_index.core.bridge.pydantic import Field, validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
 from llama_index.core.schema import Document
-
-
-nest_asyncio_err = "cannot be called from a running event loop"
-nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
-
-class ResultType(str, Enum):
-    """The result type for the parser."""
-    TXT = "text"
-    MD = "markdown"
-
-class Language(str, Enum):
-    BAZA = "abq"
-    ADYGHE = "ady"
-    AFRIKAANS = "af"
-    ANGIKA = "ang"
-    ARABIC = "ar"
-    ASSAMESE = "as"
-    AVAR = "ava"
-    AZERBAIJANI = "az"
-    BELARUSIAN = "be"
-    BULGARIAN = "bg"
-    BIHARI = "bh"
-    BHOJPURI = "bho"
-    BENGALI = "bn"
-    BOSNIAN = "bs"
-    SIMPLIFIED_CHINESE = "ch_sim"
-    TRADITIONAL_CHINESE = "ch_tra"
-    CHECHEN = "che"
-    CZECH = "cs"
-    WELSH = "cy"
-    DANISH = "da"
-    DARGWA = "dar"
-    GERMAN = "de"
-    ENGLISH = "en"
-    SPANISH = "es"
-    ESTONIAN = "et"
-    PERSIAN_FARSI = "fa"
-    FRENCH = "fr"
-    IRISH = "ga"
-    GOAN_KONKANI = "gom"
-    HINDI = "hi"
-    CROATIAN = "hr"
-    HUNGARIAN = "hu"
-    INDONESIAN = "id"
-    INGUSH = "inh"
-    ICELANDIC = "is"
-    ITALIAN = "it"
-    JAPANESE = "ja"
-    KABARDIAN = "kbd"
-    KANNADA = "kn"
-    KOREAN = "ko"
-    KURDISH = "ku"
-    LATIN = "la"
-    LAK = "lbe"
-    LEZGHIAN = "lez"
-    LITHUANIAN = "lt"
-    LATVIAN = "lv"
-    MAGAHI = "mah"
-    MAITHILI = "mai"
-    MAORI = "mi"
-    MONGOLIAN = "mn"
-    MARATHI = "mr"
-    MALAY = "ms"
-    MALTESE = "mt"
-    NEPALI = "ne"
-    NEWARI = "new"
-    DUTCH = "nl"
-    NORWEGIAN = "no"
-    OCCITAN = "oc"
-    PALI = "pi"
-    POLISH = "pl"
-    PORTUGUESE = "pt"
-    ROMANIAN = "ro"
-    RUSSIAN = "ru"
-    SERBIAN_CYRILLIC = "rs_cyrillic"
-    SERBIAN_LATIN = "rs_latin"
-    NAGPURI = "sck"
-    SLOVAK = "sk"
-    SLOVENIAN = "sl"
-    ALBANIAN = "sq"
-    SWEDISH = "sv"
-    SWAHILI = "sw"
-    TAMIL = "ta"
-    TABASSARAN = "tab"
-    TELUGU = "te"
-    THAI = "th"
-    TAJIK = "tjk"
-    TAGALOG = "tl"
-    TURKISH = "tr"
-    UYGHUR = "ug"
-    UKRANIAN = "uk"
-    URDU = "ur"
-    UZBEK = "uz"
-    VIETNAMESE = "vi"
-
-
-SUPPORTED_FILE_TYPES = [
-    ".pdf",
-    ".xml"
-    ".doc",
-    ".docx",
-    ".pptx",
-    ".rtf",
-    ".pages",
-    ".key",
-    ".epub"
-]
+from llama_parse.utils import (
+    nest_asyncio_err,
+    nest_asyncio_msg,
+    ResultType,
+    Language,
+    SUPPORTED_FILE_TYPES,
+)


 class LlamaParse(BasePydanticReader):
@@ -135,8 +34,8 @@ class LlamaParse(BasePydanticReader):
    num_workers: int = Field(
        default=4,
        gt=0,
-        lt=10, 
-        description="The number of workers to use sending API requests for parsing."
+        lt=10,
+        description="The number of workers to use sending API requests for parsing.",
    )
    check_interval: int = Field(
        default=1,
@@ -149,12 +48,18 @@ class LlamaParse(BasePydanticReader):
    verbose: bool = Field(
        default=True, description="Whether to print the progress of the parsing."
    )
-    language: Language  = Field(
-         default=Language.ENGLISH, description="The language of the text to parse."
+    show_progress: bool = Field(
+        default=True, description="Show progress when parsing multiple files."
+    )
+    language: Language = Field(
+        default=Language.ENGLISH, description="The language of the text to parse."
    )
    parsing_instruction: Optional[str] = Field(
-        default="",
-        description="The parsing instruction for the parser."
+        default="", description="The parsing instruction for the parser."
+    )
+    ignore_errors: bool = Field(
+        default=True,
+        description="Whether or not to ignore and skip errors raised during parsing.",
    )

    @validator("api_key", pre=True, always=True)
@@ -162,13 +67,14 @@ class LlamaParse(BasePydanticReader):
        """Validate the API key."""
        if not v:
            import os
+
            api_key = os.getenv("LLAMA_CLOUD_API_KEY", None)
            if api_key is None:
                raise ValueError("The API key is required.")
            return api_key
-        
+
        return v
-    
+
    @validator("base_url", pre=True, always=True)
    def validate_base_url(cls, v: str) -> str:
        """Validate the base URL."""
@@ -176,7 +82,9 @@ class LlamaParse(BasePydanticReader):
        return url or v or DEFAULT_BASE_URL

    # upload a document and get back a job_id
-    async def _create_job(self, file_path: str, extra_info: Optional[dict] = None) -> str:
+    async def _create_job(
+        self, file_path: str, extra_info: Optional[dict] = None
+    ) -> str:
        file_path = str(file_path)
        file_ext = os.path.splitext(file_path)[1]
        if file_ext not in SUPPORTED_FILE_TYPES:
@@ -198,7 +106,15 @@ class LlamaParse(BasePydanticReader):
            # send the request, start job
            url = f"{self.base_url}/api/parsing/upload"
            async with httpx.AsyncClient(timeout=self.max_timeout) as client:
-                response = await client.post(url, files=files, headers=headers, data={"language": self.language.value, "parsing_instruction": self.parsing_instruction})
+                response = await client.post(
+                    url,
+                    files=files,
+                    headers=headers,
+                    data={
+                        "language": self.language.value,
+                        "parsing_instruction": self.parsing_instruction,
+                    },
+                )
                if not response.is_success:
                    raise Exception(f"Failed to parse the file: {response.text}")

@@ -206,43 +122,65 @@ class LlamaParse(BasePydanticReader):
        job_id = response.json()["id"]
        return job_id

-    async def _get_job_result(self, job_id: str, result_type: str) -> dict:
+    async def _get_job_result(
+        self, job_id: str, result_type: str, verbose: bool = False
+    ) -> dict:
        result_url = f"{self.base_url}/api/parsing/job/{job_id}/result/{result_type}"
+        status_url = f"{self.base_url}/api/parsing/job/{job_id}"
        headers = {"Authorization": f"Bearer {self.api_key}"}

        start = time.time()
        tries = 0
        while True:
            await asyncio.sleep(self.check_interval)
-            async with httpx.AsyncClient(timeout=self.max_timeout) as client: 
-                tries += 1   
-                
-                result = await client.get(result_url, headers=headers)
+            async with httpx.AsyncClient(timeout=self.max_timeout) as client:
+                tries += 1

-                if result.status_code == 404:
+                result = await client.get(status_url, headers=headers)
+
+                if result.status_code != 200:
                    end = time.time()
                    if end - start > self.max_timeout:
-                        raise Exception(
-                            f"Timeout while parsing the file: {job_id}"
-                        )
-                    if self.verbose and tries % 10 == 0:
+                        raise Exception(f"Timeout while parsing the file: {job_id}")
+                    if verbose and tries % 10 == 0:
                        print(".", end="", flush=True)
+
+                    await asyncio.sleep(self.check_interval)
+
                    continue

-                if result.status_code == 400:
-                    detail = result.json().get("detail", "Unknown error")
-                    raise Exception(f"Failed to parse the file: {detail}")
+                # Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
+                status = result.json()["status"]
+                if status == "SUCCESS":
+                    parsed_result = await client.get(result_url, headers=headers)
+                    return parsed_result.json()
+                elif status == "PENDING":
+                    end = time.time()
+                    if end - start > self.max_timeout:
+                        raise Exception(f"Timeout while parsing the file: {job_id}")
+                    if verbose and tries % 10 == 0:
+                        print(".", end="", flush=True)

-                return result.json()
+                    await asyncio.sleep(self.check_interval)

-    async def _aload_data(self, file_path: str, extra_info: Optional[dict] = None) -> List[Document]:
+                    continue
+                else:
+                    raise Exception(
+                        f"Failed to parse the file: {job_id}, status: {status}"
+                    )
+
+    async def _aload_data(
+        self, file_path: str, extra_info: Optional[dict] = None, verbose: bool = False
+    ) -> List[Document]:
        """Load data from the input path."""
        try:
            job_id = await self._create_job(file_path, extra_info=extra_info)
-            if self.verbose:
+            if verbose:
                print("Started parsing the file under job_id %s" % job_id)
-            
-            result = await self._get_job_result(job_id, self.result_type.value)
+
+            result = await self._get_job_result(
+                job_id, self.result_type.value, verbose=verbose
+            )

            return [
                Document(
@@ -250,22 +188,39 @@ class LlamaParse(BasePydanticReader):
                    metadata=extra_info or {},
                )
            ]
-          
+
        except Exception as e:
            print(f"Error while parsing the file '{file_path}':", e)
-            raise e
-            return []
-    
+            if self.ignore_errors:
+                return []
+            else:
+                raise e

-    async def aload_data(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[Document]:
+    async def aload_data(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[Document]:
        """Load data from the input path."""
        if isinstance(file_path, (str, Path)):
-            return await self._aload_data(file_path, extra_info=extra_info)
+            return await self._aload_data(
+                file_path, extra_info=extra_info, verbose=self.verbose
+            )
        elif isinstance(file_path, list):
-            jobs = [self._aload_data(f, extra_info=extra_info) for f in file_path]
+            jobs = [
+                self._aload_data(
+                    f,
+                    extra_info=extra_info,
+                    verbose=self.verbose and not self.show_progress,
+                )
+                for f in file_path
+            ]
            try:
-                results = await run_jobs(jobs, workers=self.num_workers)
-                
+                results = await run_jobs(
+                    jobs,
+                    workers=self.num_workers,
+                    desc="Parsing files",
+                    show_progress=self.show_progress,
+                )
+
                # return flattened results
                return [item for sublist in results for item in sublist]
            except RuntimeError as e:
@@ -274,9 +229,13 @@ class LlamaParse(BasePydanticReader):
                else:
                    raise e
        else:
-            raise ValueError("The input file_path must be a string or a list of strings.")
+            raise ValueError(
+                "The input file_path must be a string or a list of strings."
+            )

-    def load_data(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[Document]:
+    def load_data(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[Document]:
        """Load data from the input path."""
        try:
            return asyncio.run(self.aload_data(file_path, extra_info))
@@ -285,35 +244,44 @@ class LlamaParse(BasePydanticReader):
                raise RuntimeError(nest_asyncio_msg)
            else:
                raise e
-    

-    async def _aget_json(self, file_path: str, extra_info: Optional[dict] = None) -> List[dict]:
+    async def _aget_json(
+        self, file_path: str, extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Load data from the input path."""
        try:
            job_id = await self._create_job(file_path, extra_info=extra_info)
            if self.verbose:
                print("Started parsing the file under job_id %s" % job_id)
-            
+
            result = await self._get_job_result(job_id, "json")
            result["job_id"] = job_id
            result["file_path"] = file_path
            return [result]
-          
+
        except Exception as e:
            print(f"Error while parsing the file '{file_path}':", e)
-            raise e
-        
-    
+            if self.ignore_errors:
+                return []
+            else:
+                raise e

-    async def aget_json(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[dict]:
+    async def aget_json(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Load data from the input path."""
        if isinstance(file_path, (str, Path)):
            return await self._aget_json(file_path, extra_info=extra_info)
        elif isinstance(file_path, list):
            jobs = [self._aget_json(f, extra_info=extra_info) for f in file_path]
            try:
-                results = await run_jobs(jobs, workers=self.num_workers)
-                
+                results = await run_jobs(
+                    jobs,
+                    workers=self.num_workers,
+                    desc="Parsing files",
+                    show_progress=self.show_progress,
+                )
+
                # return flattened results
                return [item for sublist in results for item in sublist]
            except RuntimeError as e:
@@ -322,10 +290,13 @@ class LlamaParse(BasePydanticReader):
                else:
                    raise e
        else:
-            raise ValueError("The input file_path must be a string or a list of strings.")
+            raise ValueError(
+                "The input file_path must be a string or a list of strings."
+            )

-
-    def get_json_result(self, file_path: Union[List[str], str], extra_info: Optional[dict] = None) -> List[dict]:
+    def get_json_result(
+        self, file_path: Union[List[str], str], extra_info: Optional[dict] = None
+    ) -> List[dict]:
        """Parse the input path."""
        try:
            return asyncio.run(self.aget_json(file_path, extra_info))
@@ -334,8 +305,8 @@ class LlamaParse(BasePydanticReader):
                raise RuntimeError(nest_asyncio_msg)
            else:
                raise e
-            
-    def get_images(self, json_result: list[dict], download_path: str) -> List[dict]:
+
+    def get_images(self, json_result: List[dict], download_path: str) -> List[dict]:
        """Download images from the parsed result."""
        headers = {"Authorization": f"Bearer {self.api_key}"}
        try:
@@ -347,16 +318,21 @@ class LlamaParse(BasePydanticReader):
                        print(f"> Image for page {page['page']}: {page['images']}")
                    for image in page["images"]:
                        image_name = image["name"]
-                        image_path = os.path.join(download_path, f"{job_id}-{image_name}")
-                        image["path"]=image_path
-                        image["job_id"]=job_id
-                        image["original_pdf_path"]=result["file_path"]
-                        image["page_number"]=page["page"]
+                        image_path = os.path.join(
+                            download_path, f"{job_id}-{image_name}"
+                        )
+                        image["path"] = image_path
+                        image["job_id"] = job_id
+                        image["original_pdf_path"] = result["file_path"]
+                        image["page_number"] = page["page"]
                        with open(image_path, "wb") as f:
                            image_url = f"{self.base_url}/api/parsing/job/{job_id}/result/image/{image_name}"
                            f.write(httpx.get(image_url, headers=headers).content)
                        images.append(image)
            return images
        except Exception as e:
-            print(f"Error while downloading images from the parsed result:", e)
-            return []
+            print("Error while downloading images from the parsed result:", e)
+            if self.ignore_errors:
+                return []
+            else:
+                raise e
@@ -0,0 +1,152 @@
+from enum import Enum
+
+# Asyncio error messages
+nest_asyncio_err = "cannot be called from a running event loop"
+nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
+
+
+class ResultType(str, Enum):
+    """The result type for the parser."""
+
+    TXT = "text"
+    MD = "markdown"
+    JSON = "json"
+
+
+class Language(str, Enum):
+    BAZA = "abq"
+    ADYGHE = "ady"
+    AFRIKAANS = "af"
+    ANGIKA = "ang"
+    ARABIC = "ar"
+    ASSAMESE = "as"
+    AVAR = "ava"
+    AZERBAIJANI = "az"
+    BELARUSIAN = "be"
+    BULGARIAN = "bg"
+    BIHARI = "bh"
+    BHOJPURI = "bho"
+    BENGALI = "bn"
+    BOSNIAN = "bs"
+    SIMPLIFIED_CHINESE = "ch_sim"
+    TRADITIONAL_CHINESE = "ch_tra"
+    CHECHEN = "che"
+    CZECH = "cs"
+    WELSH = "cy"
+    DANISH = "da"
+    DARGWA = "dar"
+    GERMAN = "de"
+    ENGLISH = "en"
+    SPANISH = "es"
+    ESTONIAN = "et"
+    PERSIAN_FARSI = "fa"
+    FRENCH = "fr"
+    IRISH = "ga"
+    GOAN_KONKANI = "gom"
+    HINDI = "hi"
+    CROATIAN = "hr"
+    HUNGARIAN = "hu"
+    INDONESIAN = "id"
+    INGUSH = "inh"
+    ICELANDIC = "is"
+    ITALIAN = "it"
+    JAPANESE = "ja"
+    KABARDIAN = "kbd"
+    KANNADA = "kn"
+    KOREAN = "ko"
+    KURDISH = "ku"
+    LATIN = "la"
+    LAK = "lbe"
+    LEZGHIAN = "lez"
+    LITHUANIAN = "lt"
+    LATVIAN = "lv"
+    MAGAHI = "mah"
+    MAITHILI = "mai"
+    MAORI = "mi"
+    MONGOLIAN = "mn"
+    MARATHI = "mr"
+    MALAY = "ms"
+    MALTESE = "mt"
+    NEPALI = "ne"
+    NEWARI = "new"
+    DUTCH = "nl"
+    NORWEGIAN = "no"
+    OCCITAN = "oc"
+    PALI = "pi"
+    POLISH = "pl"
+    PORTUGUESE = "pt"
+    ROMANIAN = "ro"
+    RUSSIAN = "ru"
+    SERBIAN_CYRILLIC = "rs_cyrillic"
+    SERBIAN_LATIN = "rs_latin"
+    NAGPURI = "sck"
+    SLOVAK = "sk"
+    SLOVENIAN = "sl"
+    ALBANIAN = "sq"
+    SWEDISH = "sv"
+    SWAHILI = "sw"
+    TAMIL = "ta"
+    TABASSARAN = "tab"
+    TELUGU = "te"
+    THAI = "th"
+    TAJIK = "tjk"
+    TAGALOG = "tl"
+    TURKISH = "tr"
+    UYGHUR = "ug"
+    UKRAINIAN = "uk"
+    URDU = "ur"
+    UZBEK = "uz"
+    VIETNAMESE = "vi"
+
+
+SUPPORTED_FILE_TYPES = [
+    ".pdf",
+    # Microsoft word - all versions
+    ".doc",
+    ".docx",
+    ".docm",
+    ".dot",
+    ".dotx",
+    ".dotm",
+    # Rich text format
+    ".rtf",
+    # Microsoft Works
+    ".wps",
+    # Word Perfect
+    ".wpd",
+    # Open Office
+    ".sxw",
+    ".stw",
+    ".sxg",
+    # Apple
+    ".pages",
+    # Mac Write
+    ".mw",
+    ".mcw",
+    # Unified Office Format text
+    ".uot",
+    ".uof",
+    ".uos",
+    ".uop",
+    # Microsoft powerpoints
+    ".ppt",
+    ".pptx",
+    ".pot",
+    ".pptm",
+    ".potx",
+    ".potm",
+    # Apple keynote
+    ".key",
+    # Open Office Presentations
+    ".odp",
+    ".odg",
+    ".otp",
+    ".fopd",
+    ".sxi",
+    ".sti",
+    # ebook
+    ".epub",
+    # html
+    ".html",
+    ".htm",
+]
@@ -1,6 +1,10 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "llama-parse"
-version = "0.3.9"
+version = "0.4.1"
 description = "Parse files into RAG-Optimized formats."
 authors = ["Logan Markewich <logan@llamaindex.ai>"]
 license = "MIT"
@@ -9,12 +13,8 @@ packages = [{include = "llama_parse"}]

 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
-llama-index-core = ">=0.10.7"
+llama-index-core = ">=0.10.29"

 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
 ipykernel = "^6.29.0"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
@@ -1,18 +1,59 @@
 import os
+import pytest
 from llama_parse import LlamaParse

-def test_simple_page_text():
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_text() -> None:
    parser = LlamaParse(result_type="text")

-    filepath = os.path.join(os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf")
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
    result = parser.load_data(filepath)
    assert len(result) == 1
    assert len(result[0].text) > 0

-def test_simple_page_markdown():
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_markdown() -> None:
    parser = LlamaParse(result_type="markdown")

-    filepath = os.path.join(os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf")
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
    result = parser.load_data(filepath)
    assert len(result) == 1
    assert len(result[0].text) > 0
+
+
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
+def test_simple_page_progress_workers() -> None:
+    parser = LlamaParse(result_type="markdown", show_progress=True, verbose=True)
+
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
+    result = parser.load_data([filepath, filepath])
+    assert len(result) == 2
+    assert len(result[0].text) > 0
+
+    parser = LlamaParse(
+        result_type="markdown", show_progress=True, num_workers=2, verbose=True
+    )
+
+    filepath = os.path.join(
+        os.path.dirname(__file__), "test_files/attention_is_all_you_need.pdf"
+    )
+    result = parser.load_data([filepath, filepath])
+    assert len(result) == 2
+    assert len(result[0].text) > 0
Author	SHA1	Message	Date
Pierre-Loic Doulcet	d5ca262ce6	add html support	2024-04-23 11:21:27 +08:00
Yi Ding	70b5dc3a63	some notebook updates (#148 )	2024-04-18 21:57:10 -06:00
Anoop Sharma	d6ab0aa232	Added utils files (#91 )	2024-04-14 15:17:31 -06:00
Logan	f679e1c76b	Skip tests if CICD doesn't populate the secrets (#142 )	2024-04-14 15:12:22 -06:00
Logan	b91f86ba3d	v0.4.1 (#141 )	2024-04-14 13:09:00 -06:00
Logan	0f2302fda4	QoL Changes (#140 )	2024-04-14 13:01:39 -06:00
henrycunh	ff729c05af	docs: update readme with link to llamacloud (#139 ) Having copy paste a link is silly!	2024-04-14 11:56:47 -07:00
Gautam Kumar	76a6821fb8	Fixing paths of data in example notebook (#136 )	2024-04-10 15:52:35 -06:00
Logan	97c7a38a69	vbump (#111 )	2024-03-21 10:51:50 -06:00
Pierre-Loic Doulcet	5d398a8a64	Extend supported formats (#110 )	2024-03-21 10:44:18 -06:00
Jerry Liu	4252f6186b	fixes to insurance demo (#97 )	2024-03-21 00:02:47 -07:00
Jerry Liu	22148ade9f	fix advanced RAG notebook title (#98 )	2024-03-21 00:02:38 -07:00
Jerry Liu	b8332fe8e1	nit: add colab badge to mongodb notebook (#109 )	2024-03-21 00:02:29 -07:00
Ravi Theja	e40e92a133	Add mongodb llamaparse example (#107 )	2024-03-20 23:37:42 -07:00