langgenius/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-08-24 12:32:54 -04:00
Code Issues 775 Packages Projects Releases 10 Wiki Activity

fix: handle long text URLs in webscraper (#40757)

Browse Source
This commit is contained in:
林玮 (Jade Lin)
2026-08-14 07:34:53 +00:00
committed by GitHub
parent 35132d6892
commit 4e44ade07f
2 changed files with 12 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/core/rag/extractor/extract_processor.py
+2 -1
View File
@@ -132,7 +132,6 @@ class ExtractProcessor:
storage.download(upload_file.key, file_path)
input_file = Path(file_path)
file_extension = input_file.suffix.lower()
assert upload_file is not None, "upload_file is required"
etl_type = dify_config.ETL_TYPE
extractor: BaseExtractor | None = None
if etl_type == "Unstructured":
@@ -140,6 +139,7 @@ class ExtractProcessor:
unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or ""
if file_extension in {".xlsx", ".xls"}:
assert upload_file is not None, "upload_file is required"
extractor = ExcelExtractor(
file_path,
upload_file.tenant_id,
@@ -187,6 +187,7 @@ class ExtractProcessor:
extractor = TextExtractor(file_path, autodetect_encoding=True)
else:
if file_extension in {".xlsx", ".xls"}:
assert upload_file is not None, "upload_file is required"
extractor = ExcelExtractor(
file_path,
upload_file.tenant_id,
api/tests/unit_tests/core/rag/extractor/test_extract_processor.py
+10
View File
@@ -118,6 +118,16 @@ class TestExtractProcessorLoaders:
assert len(docs) == 2
assert text == "u1\nu2"
def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch):
content = "a" * 100_000
response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode())
monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response)
monkeypatch.setattr(processor_module.dify_config, "ETL_TYPE", "SelfHosted")
text = ExtractProcessor.load_from_url("https://example.com/response.txt", return_text=True)
assert text == content
class TestExtractProcessorFileRouting:
@pytest.fixture(autouse=True)