mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 21:33:31 -04:00
Leonid Ganeline
b96ab4b763
# Docs: improvements in the `retrievers/examples/` notebooks Its primary purpose is to make the Jupyter notebook examples **consistent** and more suitable for first-time viewers. - add links to the integration source (if applicable) with a short description of this source; - removed `_retriever` suffix from the file names (where it existed) for consistency; - removed ` retriever` from the notebook title (where it existed) for consistency; - added code to install necessary Python package(s); - added code to set up the necessary API Key. - very small fixes in notebooks from other folders (for consistency): - docs/modules/indexes/vectorstores/examples/elasticsearch.ipynb - docs/modules/indexes/vectorstores/examples/pinecone.ipynb - docs/modules/models/llms/integrations/cohere.ipynb - fixed misspelling in langchain/retrievers/time_weighted_retriever.py comment (sorry, about this change in a .py file ) ## Who can review @dev2049
88 lines
2.7 KiB
Python
88 lines
2.7 KiB
Python
"""Abstract interface for document loader implementations."""
|
|
from abc import ABC, abstractmethod
|
|
from typing import Iterator, List, Optional
|
|
|
|
from langchain.document_loaders.blob_loaders import Blob
|
|
from langchain.schema import Document
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter, TextSplitter
|
|
|
|
|
|
class BaseLoader(ABC):
|
|
"""Interface for loading documents.
|
|
|
|
Implementations should implement the lazy-loading method using generators
|
|
to avoid loading all documents into memory at once.
|
|
|
|
The `load` method will remain as is for backwards compatibility, but its
|
|
implementation should be just `list(self.lazy_load())`.
|
|
"""
|
|
|
|
# Sub-classes should implement this method
|
|
# as return list(self.lazy_load()).
|
|
# This method returns a List which is materialized in memory.
|
|
@abstractmethod
|
|
def load(self) -> List[Document]:
|
|
"""Load data into document objects."""
|
|
|
|
def load_and_split(
|
|
self, text_splitter: Optional[TextSplitter] = None
|
|
) -> List[Document]:
|
|
"""Load documents and split into chunks."""
|
|
if text_splitter is None:
|
|
_text_splitter: TextSplitter = RecursiveCharacterTextSplitter()
|
|
else:
|
|
_text_splitter = text_splitter
|
|
docs = self.load()
|
|
return _text_splitter.split_documents(docs)
|
|
|
|
# Attention: This method will be upgraded into an abstractmethod once it's
|
|
# implemented in all the existing subclasses.
|
|
def lazy_load(
|
|
self,
|
|
) -> Iterator[Document]:
|
|
"""A lazy loader for document content."""
|
|
raise NotImplementedError(
|
|
f"{self.__class__.__name__} does not implement lazy_load()"
|
|
)
|
|
|
|
|
|
class BaseBlobParser(ABC):
|
|
"""Abstract interface for blob parsers.
|
|
|
|
A blob parser is provides a way to parse raw data stored in a blob into one
|
|
or more documents.
|
|
|
|
The parser can be composed with blob loaders, making it easy to re-use
|
|
a parser independent of how the blob was originally loaded.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
|
"""Lazy parsing interface.
|
|
|
|
Subclasses are required to implement this method.
|
|
|
|
Args:
|
|
blob: Blob instance
|
|
|
|
Returns:
|
|
Generator of documents
|
|
"""
|
|
|
|
def parse(self, blob: Blob) -> List[Document]:
|
|
"""Eagerly parse the blob into a document or documents.
|
|
|
|
This is a convenience method for interactive development environment.
|
|
|
|
Production applications should favor the lazy_parse method instead.
|
|
|
|
Subclasses should generally not over-ride this parse method.
|
|
|
|
Args:
|
|
blob: Blob instance
|
|
|
|
Returns:
|
|
List of documents
|
|
"""
|
|
return list(self.lazy_parse(blob))
|