mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 21:33:31 -04:00
Lance Martin
6e69bfbb28
Many cities have open data portals for events like crime, traffic, etc. Socrata provides an API for many, including SF (e.g., see [here](https://dev.socrata.com/foundry/data.sfgov.org/tmnf-yvry)). This is a new data loader for city data that uses Socrata API.
38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
from typing import Iterator, List
|
|
|
|
from langchain.docstore.document import Document
|
|
from langchain.document_loaders.base import BaseLoader
|
|
|
|
|
|
class OpenCityDataLoader(BaseLoader):
|
|
"""Loader that loads Open city data."""
|
|
|
|
def __init__(self, city_id: str, dataset_id: str, limit: int):
|
|
"""Initialize with dataset_id"""
|
|
""" Example: https://dev.socrata.com/foundry/data.sfgov.org/vw6y-z8j6 """
|
|
""" e.g., city_id = data.sfgov.org """
|
|
""" e.g., dataset_id = vw6y-z8j6 """
|
|
self.city_id = city_id
|
|
self.dataset_id = dataset_id
|
|
self.limit = limit
|
|
|
|
def lazy_load(self) -> Iterator[Document]:
|
|
"""Lazy load records."""
|
|
|
|
from sodapy import Socrata
|
|
|
|
client = Socrata(self.city_id, None)
|
|
results = client.get(self.dataset_id, limit=self.limit)
|
|
for record in results:
|
|
yield Document(
|
|
page_content=str(record),
|
|
metadata={
|
|
"source": self.city_id + "_" + self.dataset_id,
|
|
},
|
|
)
|
|
|
|
def load(self) -> List[Document]:
|
|
"""Load records."""
|
|
|
|
return list(self.lazy_load())
|