mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-08-27 01:15:22 -04:00
Matt Robinson
63aa28e2a6
### Summary
Allows users to pass in `**unstructured_kwargs` to Unstructured document
loaders. Implemented with the `strategy` kwargs in mind, but will pass
in other kwargs like `include_page_breaks` as well. The two currently
supported strategies are `"hi_res"`, which is more accurate but takes
longer, and `"fast"`, which processes faster but with lower accuracy.
The `"hi_res"` strategy is the default. For PDFs, if `detectron2` is not
available and the user selects `"hi_res"`, the loader will fallback to
using the `"fast"` strategy.
### Testing
#### Make sure the `strategy` kwarg works
Run the following in iPython to verify that the `"fast"` strategy is
indeed faster.
```python
from langchain.document_loaders import UnstructuredFileLoader
loader = UnstructuredFileLoader("layout-parser-paper-fast.pdf", strategy="fast", mode="elements")
%timeit loader.load()
loader = UnstructuredFileLoader("layout-parser-paper-fast.pdf", mode="elements")
%timeit loader.load()
```
On my system I get:
```python
In [3]: from langchain.document_loaders import UnstructuredFileLoader
In [4]: loader = UnstructuredFileLoader("layout-parser-paper-fast.pdf", strategy="fast", mode="elements")
In [5]: %timeit loader.load()
247 ms ± 369 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)
In [6]: loader = UnstructuredFileLoader("layout-parser-paper-fast.pdf", mode="elements")
In [7]: %timeit loader.load()
2.45 s ± 31 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
```
#### Make sure older versions of `unstructured` still work
Run `pip install unstructured==0.5.3` and then verify the following runs
without error:
```python
from langchain.document_loaders import UnstructuredFileLoader
loader = UnstructuredFileLoader("layout-parser-paper-fast.pdf", mode="elements")
loader.load()
```
44 lines
1.7 KiB
Python
44 lines
1.7 KiB
Python
"""Loader that loads powerpoint files."""
|
|
import os
|
|
from typing import List
|
|
|
|
from langchain.document_loaders.unstructured import UnstructuredFileLoader
|
|
|
|
|
|
class UnstructuredPowerPointLoader(UnstructuredFileLoader):
|
|
"""Loader that uses unstructured to load powerpoint files."""
|
|
|
|
def _get_elements(self) -> List:
|
|
from unstructured.__version__ import __version__ as __unstructured_version__
|
|
from unstructured.file_utils.filetype import FileType, detect_filetype
|
|
|
|
unstructured_version = tuple(
|
|
[int(x) for x in __unstructured_version__.split(".")]
|
|
)
|
|
# NOTE(MthwRobinson) - magic will raise an import error if the libmagic
|
|
# system dependency isn't installed. If it's not installed, we'll just
|
|
# check the file extension
|
|
try:
|
|
import magic # noqa: F401
|
|
|
|
is_ppt = detect_filetype(self.file_path) == FileType.PPT
|
|
except ImportError:
|
|
_, extension = os.path.splitext(self.file_path)
|
|
is_ppt = extension == ".ppt"
|
|
|
|
if is_ppt and unstructured_version < (0, 4, 11):
|
|
raise ValueError(
|
|
f"You are on unstructured version {__unstructured_version__}. "
|
|
"Partitioning .ppt files is only supported in unstructured>=0.4.11. "
|
|
"Please upgrade the unstructured package and try again."
|
|
)
|
|
|
|
if is_ppt:
|
|
from unstructured.partition.ppt import partition_ppt
|
|
|
|
return partition_ppt(filename=self.file_path, **self.unstructured_kwargs)
|
|
else:
|
|
from unstructured.partition.pptx import partition_pptx
|
|
|
|
return partition_pptx(filename=self.file_path, **self.unstructured_kwargs)
|