mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 13:26:32 -04:00
Paul Garner
69698be3e6
re https://github.com/hwchase17/langchain/issues/439#issuecomment-1510442791 I think it's not polite for a library to use the root logger both of these forms are also used: ``` logger = logging.getLogger(__name__) logger = logging.getLogger(__file__) ``` I am not sure if there is any reason behind one vs the other? (...I am guessing maybe just contributed by different people) it seems to me it'd be better to consistently use `logging.getLogger(__name__)` this makes it easier for consumers of the library to set up log handlers, e.g. for everything with `langchain.` prefix
56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
"""Loader that uses Diffbot to load webpages in text format."""
|
|
import logging
|
|
from typing import Any, List
|
|
|
|
import requests
|
|
|
|
from langchain.docstore.document import Document
|
|
from langchain.document_loaders.base import BaseLoader
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DiffbotLoader(BaseLoader):
|
|
"""Loader that loads Diffbot file json."""
|
|
|
|
def __init__(
|
|
self, api_token: str, urls: List[str], continue_on_failure: bool = True
|
|
):
|
|
"""Initialize with API token, ids, and key."""
|
|
self.api_token = api_token
|
|
self.urls = urls
|
|
self.continue_on_failure = continue_on_failure
|
|
|
|
def _diffbot_api_url(self, diffbot_api: str) -> str:
|
|
return f"https://api.diffbot.com/v3/{diffbot_api}"
|
|
|
|
def _get_diffbot_data(self, url: str) -> Any:
|
|
"""Get Diffbot file from Diffbot REST API."""
|
|
# TODO: Add support for other Diffbot APIs
|
|
diffbot_url = self._diffbot_api_url("article")
|
|
params = {
|
|
"token": self.api_token,
|
|
"url": url,
|
|
}
|
|
response = requests.get(diffbot_url, params=params, timeout=10)
|
|
|
|
# TODO: handle non-ok errors
|
|
return response.json() if response.ok else {}
|
|
|
|
def load(self) -> List[Document]:
|
|
"""Extract text from Diffbot on all the URLs and return Document instances"""
|
|
docs: List[Document] = list()
|
|
|
|
for url in self.urls:
|
|
try:
|
|
data = self._get_diffbot_data(url)
|
|
text = data["objects"][0]["text"] if "objects" in data else ""
|
|
metadata = {"source": url}
|
|
docs.append(Document(page_content=text, metadata=metadata))
|
|
except Exception as e:
|
|
if self.continue_on_failure:
|
|
logger.error(f"Error fetching or processing {url}, exception: {e}")
|
|
else:
|
|
raise e
|
|
return docs
|