mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 13:26:32 -04:00
Matt Robinson
270384fb44
### Summary #1667 updated several Unstructured loaders to accept `unstructured_kwargs` in the `__init__` function. However, the previous PR did not add this functionality to every Unstructured loader. This PR ensures `unstructured_kwargs` are passed in all remaining Unstructured loaders.
26 lines
1.1 KiB
Python
26 lines
1.1 KiB
Python
"""Loader that loads Markdown files."""
|
|
from typing import List
|
|
|
|
from langchain.document_loaders.unstructured import UnstructuredFileLoader
|
|
|
|
|
|
class UnstructuredMarkdownLoader(UnstructuredFileLoader):
|
|
"""Loader that uses unstructured to load markdown files."""
|
|
|
|
def _get_elements(self) -> List:
|
|
from unstructured.__version__ import __version__ as __unstructured_version__
|
|
from unstructured.partition.md import partition_md
|
|
|
|
# NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
|
|
# versions of unstructured like 0.4.17-dev1
|
|
_unstructured_version = __unstructured_version__.split("-")[0]
|
|
unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
|
|
|
|
if unstructured_version < (0, 4, 16):
|
|
raise ValueError(
|
|
f"You are on unstructured version {__unstructured_version__}. "
|
|
"Partitioning markdown files is only supported in unstructured>=0.4.16."
|
|
)
|
|
|
|
return partition_md(filename=self.file_path, **self.unstructured_kwargs)
|