mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-08-27 01:15:22 -04:00
Paul Garner
69698be3e6
re https://github.com/hwchase17/langchain/issues/439#issuecomment-1510442791 I think it's not polite for a library to use the root logger both of these forms are also used: ``` logger = logging.getLogger(__name__) logger = logging.getLogger(__file__) ``` I am not sure if there is any reason behind one vs the other? (...I am guessing maybe just contributed by different people) it seems to me it'd be better to consistently use `logging.getLogger(__name__)` this makes it easier for consumers of the library to set up log handlers, e.g. for everything with `langchain.` prefix
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
"""Loading logic for loading documents from a directory."""
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import List, Type, Union
|
|
|
|
from langchain.docstore.document import Document
|
|
from langchain.document_loaders.base import BaseLoader
|
|
from langchain.document_loaders.html_bs import BSHTMLLoader
|
|
from langchain.document_loaders.text import TextLoader
|
|
from langchain.document_loaders.unstructured import UnstructuredFileLoader
|
|
|
|
FILE_LOADER_TYPE = Union[
|
|
Type[UnstructuredFileLoader], Type[TextLoader], Type[BSHTMLLoader]
|
|
]
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _is_visible(p: Path) -> bool:
|
|
parts = p.parts
|
|
for _p in parts:
|
|
if _p.startswith("."):
|
|
return False
|
|
return True
|
|
|
|
|
|
class DirectoryLoader(BaseLoader):
|
|
"""Loading logic for loading documents from a directory."""
|
|
|
|
def __init__(
|
|
self,
|
|
path: str,
|
|
glob: str = "**/[!.]*",
|
|
silent_errors: bool = False,
|
|
load_hidden: bool = False,
|
|
loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
|
|
loader_kwargs: Union[dict, None] = None,
|
|
recursive: bool = False,
|
|
):
|
|
"""Initialize with path to directory and how to glob over it."""
|
|
if loader_kwargs is None:
|
|
loader_kwargs = {}
|
|
self.path = path
|
|
self.glob = glob
|
|
self.load_hidden = load_hidden
|
|
self.loader_cls = loader_cls
|
|
self.loader_kwargs = loader_kwargs
|
|
self.silent_errors = silent_errors
|
|
self.recursive = recursive
|
|
|
|
def load(self) -> List[Document]:
|
|
"""Load documents."""
|
|
p = Path(self.path)
|
|
docs = []
|
|
items = p.rglob(self.glob) if self.recursive else p.glob(self.glob)
|
|
for i in items:
|
|
if i.is_file():
|
|
if _is_visible(i.relative_to(p)) or self.load_hidden:
|
|
try:
|
|
sub_docs = self.loader_cls(str(i), **self.loader_kwargs).load()
|
|
docs.extend(sub_docs)
|
|
except Exception as e:
|
|
if self.silent_errors:
|
|
logger.warning(e)
|
|
else:
|
|
raise e
|
|
return docs
|