From 4e44ade07f1fd70563f22d465f20cb9983fd8a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E7=8E=AE=20=28Jade=20Lin=29?= Date: Fri, 14 Aug 2026 07:34:53 +0000 Subject: [PATCH] fix: handle long text URLs in webscraper (#40757) --- api/core/rag/extractor/extract_processor.py | 3 ++- .../core/rag/extractor/test_extract_processor.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 7eed135bbf0..0a008b5f31e 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -132,7 +132,6 @@ class ExtractProcessor: storage.download(upload_file.key, file_path) input_file = Path(file_path) file_extension = input_file.suffix.lower() - assert upload_file is not None, "upload_file is required" etl_type = dify_config.ETL_TYPE extractor: BaseExtractor | None = None if etl_type == "Unstructured": @@ -140,6 +139,7 @@ class ExtractProcessor: unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY or "" if file_extension in {".xlsx", ".xls"}: + assert upload_file is not None, "upload_file is required" extractor = ExcelExtractor( file_path, upload_file.tenant_id, @@ -187,6 +187,7 @@ class ExtractProcessor: extractor = TextExtractor(file_path, autodetect_encoding=True) else: if file_extension in {".xlsx", ".xls"}: + assert upload_file is not None, "upload_file is required" extractor = ExcelExtractor( file_path, upload_file.tenant_id, diff --git a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py index 19d16e1d552..0b73d488fa0 100644 --- a/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py +++ b/api/tests/unit_tests/core/rag/extractor/test_extract_processor.py @@ -118,6 +118,16 @@ class TestExtractProcessorLoaders: assert len(docs) == 2 assert text == "u1\nu2" + def test_load_from_url_extracts_long_text_without_upload_file(self, monkeypatch: pytest.MonkeyPatch): + content = "a" * 100_000 + response = SimpleNamespace(headers={"Content-Type": "text/plain"}, content=content.encode()) + monkeypatch.setattr(processor_module.remote_fetcher, "make_request", lambda *args, **kwargs: response) + monkeypatch.setattr(processor_module.dify_config, "ETL_TYPE", "SelfHosted") + + text = ExtractProcessor.load_from_url("https://example.com/response.txt", return_text=True) + + assert text == content + class TestExtractProcessorFileRouting: @pytest.fixture(autouse=True)