mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 21:33:31 -04:00
Maciej Bryński
aa345a4bb7
By default get_text doesn't separate content of different HTML tag. Adding option for specifying separator helps with document splitting.
58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
"""Loader that uses bs4 to load HTML files, enriching metadata with page title."""
|
|
|
|
import logging
|
|
from typing import Dict, List, Union
|
|
|
|
from langchain.docstore.document import Document
|
|
from langchain.document_loaders.base import BaseLoader
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BSHTMLLoader(BaseLoader):
|
|
"""Loader that uses beautiful soup to parse HTML files."""
|
|
|
|
def __init__(
|
|
self,
|
|
file_path: str,
|
|
open_encoding: Union[str, None] = None,
|
|
bs_kwargs: Union[dict, None] = None,
|
|
get_text_separator: str = "",
|
|
) -> None:
|
|
"""Initialise with path, and optionally, file encoding to use, and any kwargs
|
|
to pass to the BeautifulSoup object."""
|
|
try:
|
|
import bs4 # noqa:F401
|
|
except ImportError:
|
|
raise ValueError(
|
|
"beautifulsoup4 package not found, please install it with "
|
|
"`pip install beautifulsoup4`"
|
|
)
|
|
|
|
self.file_path = file_path
|
|
self.open_encoding = open_encoding
|
|
if bs_kwargs is None:
|
|
bs_kwargs = {"features": "lxml"}
|
|
self.bs_kwargs = bs_kwargs
|
|
self.get_text_separator = get_text_separator
|
|
|
|
def load(self) -> List[Document]:
|
|
from bs4 import BeautifulSoup
|
|
|
|
"""Load HTML document into document objects."""
|
|
with open(self.file_path, "r", encoding=self.open_encoding) as f:
|
|
soup = BeautifulSoup(f, **self.bs_kwargs)
|
|
|
|
text = soup.get_text(self.get_text_separator)
|
|
|
|
if soup.title:
|
|
title = str(soup.title.string)
|
|
else:
|
|
title = ""
|
|
|
|
metadata: Dict[str, Union[str, None]] = {
|
|
"source": self.file_path,
|
|
"title": title,
|
|
}
|
|
return [Document(page_content=text, metadata=metadata)]
|