add html support

examples/demo_advanced.ipynb

@@ -19,12 +19,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
    ]
   },
   {

examples/demo_advanced_astradb.ipynb

@@ -17,13 +17,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core==0.10.6.post1\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse\n",
-    "!pip install llama-index-vector-stores-astra-db"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core==0.10.6.post1\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse\n",
+    "%pip install llama-index-vector-stores-astra-db"
    ]
   },
   {

examples/demo_astradb.ipynb

@@ -28,7 +28,7 @@
    "outputs": [],
    "source": [
     "# First, install the required dependencies\n",
-    "!pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
+    "%pip install --quiet llama-index llama-parse llama-index-vector-stores-astra-db llama-index-llms-openai"
    ]
   },
   {

examples/demo_basic.ipynb

@@ -13,7 +13,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
    ]
   },
   {

examples/demo_insurance.ipynb

@@ -24,7 +24,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
    ]
   },
   {

examples/demo_json.ipynb

@@ -206,7 +206,7 @@
     "    for image_dict in image_dicts:\n",
     "        image_doc = ImageDocument(image_path=image_dict[\"path\"])\n",
     "        response = anthropic_mm_llm.complete(\n",
-    "            prompt=\"Describe the images as an alternative text\",\n",
+    "            prompt=\"Describe the images as alt text\",\n",
     "            image_documents=[image_doc],\n",
     "        )\n",
     "        text_node = TextNode(text=str(response), metadata={\"path\": image_dict[\"path\"]})\n",

examples/demo_json_parsing.ipynb

@@ -33,11 +33,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-llms-anthropic llama-index-multi-modal-llms-anthropic\n",
-    "!pip install llama-index-embeddings-huggingface\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-llms-anthropic llama-index-multi-modal-llms-anthropic\n",
+    "%pip install llama-index-embeddings-huggingface\n",
+    "%pip install llama-parse"
    ]
   },
   {

examples/demo_languages.ipynb

@@ -21,7 +21,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index llama-parse"
+    "%pip install llama-index llama-parse"
    ]
   },
   {

examples/demo_mongodb.ipynb

@@ -29,7 +29,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index llama-parse pip install llama-index-vector-stores-mongodb llama-index-llms-openai"
+    "%pip install llama-index llama-parse\n",
+    "%pip install llama-index-vector-stores-mongodb llama-index-llms-openai"
    ]
   },
   {

examples/demo_parsing_instructions.ipynb

@@ -114,7 +114,7 @@
     }
    ],
    "source": [
-    "!pip install llama-parse"
+    "%pip install llama-parse"
    ]
   },
   {
@@ -294,7 +294,7 @@
     "### Using parsing instructions\n",
     "Let's try to parse the manga with custom instructions:\n",
     "\n",
-    "\"The provided document is a manga comic book. Most pages do NOT have title. It does not contain tables. Try to reconstruct the dialogue happening in a cohesive way.\"\n",
+    "\"The provided document is a manga comic book. Most pages do NOT have a title. It does not contain tables. Try to reconstruct the dialogue spoken in a cohesive way.\"\n",
     "\n",
     "To do so just pass the parsing instruction as a parameter to LlamaParse:"
    ]
@@ -314,9 +314,9 @@
     }
    ],
    "source": [
-    "parsingInstructionManga = \"\"\"The provided document is a manga comic book, most page do NOT have title.\n",
-    "It does not contain table.\n",
-    "Try to reconstruct the dialog happening in a cohesive way.\"\"\"\n",
+    "parsingInstructionManga = \"\"\"The provided document is a manga comic book. Most pages do NOT have a title.\n",
+    "It does not contain tables.\n",
+    "Try to reconstruct the dialogue spoken in a cohesive way.\"\"\"\n",
     "withInstructionParsing = LlamaParse(\n",
     "    result_type=\"markdown\", parsing_instruction=parsingInstructionManga\n",
     ").load_data(\"./manga.pdf\")"
@@ -404,9 +404,9 @@
     }
    ],
    "source": [
-    "parsingInstructionMangaLatex = \"\"\"The provided document is a manga comic book, most page do NOT have title.\n",
-    "It does not contain table. Do not output table.\n",
-    "Try to reconstruct the dialog happening in a cohesive way.\n",
+    "parsingInstructionMangaLatex = \"\"\"The provided document is a manga comic book. Most pages do NOT have a title.\n",
+    "It does not contain tables.\n",
+    "Try to reconstruct the dialogue spoken in a cohesive way.\n",
     "Output any math equation in LATEX markdown (between $$)\"\"\"\n",
     "withLatex = LlamaParse(\n",
     "    result_type=\"markdown\", parsing_instruction=parsingInstructionMangaLatex\n",

examples/demo_table_comparisons.ipynb

@@ -28,13 +28,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-index-core\n",
-    "!pip install llama-index-embeddings-openai\n",
-    "!pip install llama-index-question-gen-openai\n",
-    "!pip install llama-index-postprocessor-flag-embedding-reranker\n",
-    "!pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
-    "!pip install llama-parse"
+    "%pip install llama-index\n",
+    "%pip install llama-index-core\n",
+    "%pip install llama-index-embeddings-openai\n",
+    "%pip install llama-index-question-gen-openai\n",
+    "%pip install llama-index-postprocessor-flag-embedding-reranker\n",
+    "%pip install git+https://github.com/FlagOpen/FlagEmbedding.git\n",
+    "%pip install llama-parse"
    ]
   },
   {

examples/other_files/demo_ppt_financial.ipynb

@@ -26,9 +26,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install llama-index\n",
-    "!pip install llama-parse\n",
-    "!pip install torch transformers python-pptx Pillow"
+    "%pip install llama-index\n",
+    "%pip install llama-parse\n",
+    "%pip install torch transformers python-pptx Pillow"
    ]
   },
   {

llama_parse/base.py

@@ -3,7 +3,6 @@ import asyncio
 import httpx
 import mimetypes
 import time
-from enum import Enum
 from pathlib import Path
 from typing import List, Optional, Union
 
@@ -12,154 +11,13 @@ from llama_index.core.bridge.pydantic import Field, validator
 from llama_index.core.constants import DEFAULT_BASE_URL
 from llama_index.core.readers.base import BasePydanticReader
 from llama_index.core.schema import Document
-
-
-nest_asyncio_err = "cannot be called from a running event loop"
-nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
-
-
-class ResultType(str, Enum):
-    """The result type for the parser."""
-
-    TXT = "text"
-    MD = "markdown"
-    JSON = "json"
-
-
-class Language(str, Enum):
-    BAZA = "abq"
-    ADYGHE = "ady"
-    AFRIKAANS = "af"
-    ANGIKA = "ang"
-    ARABIC = "ar"
-    ASSAMESE = "as"
-    AVAR = "ava"
-    AZERBAIJANI = "az"
-    BELARUSIAN = "be"
-    BULGARIAN = "bg"
-    BIHARI = "bh"
-    BHOJPURI = "bho"
-    BENGALI = "bn"
-    BOSNIAN = "bs"
-    SIMPLIFIED_CHINESE = "ch_sim"
-    TRADITIONAL_CHINESE = "ch_tra"
-    CHECHEN = "che"
-    CZECH = "cs"
-    WELSH = "cy"
-    DANISH = "da"
-    DARGWA = "dar"
-    GERMAN = "de"
-    ENGLISH = "en"
-    SPANISH = "es"
-    ESTONIAN = "et"
-    PERSIAN_FARSI = "fa"
-    FRENCH = "fr"
-    IRISH = "ga"
-    GOAN_KONKANI = "gom"
-    HINDI = "hi"
-    CROATIAN = "hr"
-    HUNGARIAN = "hu"
-    INDONESIAN = "id"
-    INGUSH = "inh"
-    ICELANDIC = "is"
-    ITALIAN = "it"
-    JAPANESE = "ja"
-    KABARDIAN = "kbd"
-    KANNADA = "kn"
-    KOREAN = "ko"
-    KURDISH = "ku"
-    LATIN = "la"
-    LAK = "lbe"
-    LEZGHIAN = "lez"
-    LITHUANIAN = "lt"
-    LATVIAN = "lv"
-    MAGAHI = "mah"
-    MAITHILI = "mai"
-    MAORI = "mi"
-    MONGOLIAN = "mn"
-    MARATHI = "mr"
-    MALAY = "ms"
-    MALTESE = "mt"
-    NEPALI = "ne"
-    NEWARI = "new"
-    DUTCH = "nl"
-    NORWEGIAN = "no"
-    OCCITAN = "oc"
-    PALI = "pi"
-    POLISH = "pl"
-    PORTUGUESE = "pt"
-    ROMANIAN = "ro"
-    RUSSIAN = "ru"
-    SERBIAN_CYRILLIC = "rs_cyrillic"
-    SERBIAN_LATIN = "rs_latin"
-    NAGPURI = "sck"
-    SLOVAK = "sk"
-    SLOVENIAN = "sl"
-    ALBANIAN = "sq"
-    SWEDISH = "sv"
-    SWAHILI = "sw"
-    TAMIL = "ta"
-    TABASSARAN = "tab"
-    TELUGU = "te"
-    THAI = "th"
-    TAJIK = "tjk"
-    TAGALOG = "tl"
-    TURKISH = "tr"
-    UYGHUR = "ug"
-    UKRAINIAN = "uk"
-    URDU = "ur"
-    UZBEK = "uz"
-    VIETNAMESE = "vi"
-
-
-SUPPORTED_FILE_TYPES = [
-    ".pdf",
-    # Microsoft word - all versions
-    ".doc",
-    ".docx",
-    ".docm",
-    ".dot",
-    ".dotx",
-    ".dotm",
-    # Rich text format
-    ".rtf",
-    # Microsoft Works
-    ".wps",
-    # Word Perfect
-    ".wpd",
-    # Open Office
-    ".sxw",
-    ".stw",
-    ".sxg",
-    # Apple
-    ".pages",
-    # Mac Write
-    ".mw",
-    ".mcw",
-    # Unified Office Format text
-    ".uot",
-    ".uof",
-    ".uos",
-    ".uop",
-    # Microsoft powerpoints
-    ".ppt",
-    ".pptx",
-    ".pot",
-    ".pptm",
-    ".potx",
-    ".potm",
-    # Apple keynote
-    ".key",
-    # Open Office Presentations
-    ".odp",
-    ".odg",
-    ".otp",
-    ".fopd",
-    ".sxi",
-    ".sti",
-    # ebook
-    ".epub",
-]
+from llama_parse.utils import (
+    nest_asyncio_err,
+    nest_asyncio_msg,
+    ResultType,
+    Language,
+    SUPPORTED_FILE_TYPES,
+)
 
 
 class LlamaParse(BasePydanticReader):

llama_parse/utils.py

@@ -0,0 +1,152 @@
+from enum import Enum
+
+# Asyncio error messages
+nest_asyncio_err = "cannot be called from a running event loop"
+nest_asyncio_msg = "The event loop is already running. Add `import nest_asyncio; nest_asyncio.apply()` to your code to fix this issue."
+
+
+class ResultType(str, Enum):
+    """The result type for the parser."""
+
+    TXT = "text"
+    MD = "markdown"
+    JSON = "json"
+
+
+class Language(str, Enum):
+    BAZA = "abq"
+    ADYGHE = "ady"
+    AFRIKAANS = "af"
+    ANGIKA = "ang"
+    ARABIC = "ar"
+    ASSAMESE = "as"
+    AVAR = "ava"
+    AZERBAIJANI = "az"
+    BELARUSIAN = "be"
+    BULGARIAN = "bg"
+    BIHARI = "bh"
+    BHOJPURI = "bho"
+    BENGALI = "bn"
+    BOSNIAN = "bs"
+    SIMPLIFIED_CHINESE = "ch_sim"
+    TRADITIONAL_CHINESE = "ch_tra"
+    CHECHEN = "che"
+    CZECH = "cs"
+    WELSH = "cy"
+    DANISH = "da"
+    DARGWA = "dar"
+    GERMAN = "de"
+    ENGLISH = "en"
+    SPANISH = "es"
+    ESTONIAN = "et"
+    PERSIAN_FARSI = "fa"
+    FRENCH = "fr"
+    IRISH = "ga"
+    GOAN_KONKANI = "gom"
+    HINDI = "hi"
+    CROATIAN = "hr"
+    HUNGARIAN = "hu"
+    INDONESIAN = "id"
+    INGUSH = "inh"
+    ICELANDIC = "is"
+    ITALIAN = "it"
+    JAPANESE = "ja"
+    KABARDIAN = "kbd"
+    KANNADA = "kn"
+    KOREAN = "ko"
+    KURDISH = "ku"
+    LATIN = "la"
+    LAK = "lbe"
+    LEZGHIAN = "lez"
+    LITHUANIAN = "lt"
+    LATVIAN = "lv"
+    MAGAHI = "mah"
+    MAITHILI = "mai"
+    MAORI = "mi"
+    MONGOLIAN = "mn"
+    MARATHI = "mr"
+    MALAY = "ms"
+    MALTESE = "mt"
+    NEPALI = "ne"
+    NEWARI = "new"
+    DUTCH = "nl"
+    NORWEGIAN = "no"
+    OCCITAN = "oc"
+    PALI = "pi"
+    POLISH = "pl"
+    PORTUGUESE = "pt"
+    ROMANIAN = "ro"
+    RUSSIAN = "ru"
+    SERBIAN_CYRILLIC = "rs_cyrillic"
+    SERBIAN_LATIN = "rs_latin"
+    NAGPURI = "sck"
+    SLOVAK = "sk"
+    SLOVENIAN = "sl"
+    ALBANIAN = "sq"
+    SWEDISH = "sv"
+    SWAHILI = "sw"
+    TAMIL = "ta"
+    TABASSARAN = "tab"
+    TELUGU = "te"
+    THAI = "th"
+    TAJIK = "tjk"
+    TAGALOG = "tl"
+    TURKISH = "tr"
+    UYGHUR = "ug"
+    UKRAINIAN = "uk"
+    URDU = "ur"
+    UZBEK = "uz"
+    VIETNAMESE = "vi"
+
+
+SUPPORTED_FILE_TYPES = [
+    ".pdf",
+    # Microsoft word - all versions
+    ".doc",
+    ".docx",
+    ".docm",
+    ".dot",
+    ".dotx",
+    ".dotm",
+    # Rich text format
+    ".rtf",
+    # Microsoft Works
+    ".wps",
+    # Word Perfect
+    ".wpd",
+    # Open Office
+    ".sxw",
+    ".stw",
+    ".sxg",
+    # Apple
+    ".pages",
+    # Mac Write
+    ".mw",
+    ".mcw",
+    # Unified Office Format text
+    ".uot",
+    ".uof",
+    ".uos",
+    ".uop",
+    # Microsoft powerpoints
+    ".ppt",
+    ".pptx",
+    ".pot",
+    ".pptm",
+    ".potx",
+    ".potm",
+    # Apple keynote
+    ".key",
+    # Open Office Presentations
+    ".odp",
+    ".odg",
+    ".otp",
+    ".fopd",
+    ".sxi",
+    ".sti",
+    # ebook
+    ".epub",
+    # html
+    ".html",
+    ".htm",
+]

tests/test_reader.py

@@ -1,7 +1,12 @@
 import os
+import pytest
 from llama_parse import LlamaParse
 
 
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
 def test_simple_page_text() -> None:
     parser = LlamaParse(result_type="text")
 
@@ -13,6 +18,10 @@ def test_simple_page_text() -> None:
     assert len(result[0].text) > 0
 
 
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
 def test_simple_page_markdown() -> None:
     parser = LlamaParse(result_type="markdown")
 
@@ -24,6 +33,10 @@ def test_simple_page_markdown() -> None:
     assert len(result[0].text) > 0
 
 
+@pytest.mark.skipif(
+    os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
+    reason="LLAMA_CLOUD_API_KEY not set",
+)
 def test_simple_page_progress_workers() -> None:
     parser = LlamaParse(result_type="markdown", show_progress=True, verbose=True)