Fix: parse pdf in chat (#18860)

rag/app/naive.py

@@ -470,7 +470,7 @@ def by_mistral_ocr(
 
 def by_plaintext(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=None, **kwargs):
     layout_recognizer = (kwargs.get("layout_recognizer") or "").strip()
-    if (not layout_recognizer) or (layout_recognizer == "Plain Text"):
+    if (not layout_recognizer) or layout_recognizer.replace(" ", "").lower() == "plaintext":
         pdf_parser = PlainParser()
     else:
         tenant_id = kwargs.get("tenant_id")

run_go_tests.sh

@@ -27,4 +27,4 @@ for pkg in "${PACKAGES[@]}"; do
 done
 
 #echo "Running all tests except failed packages..."
-#go test $(go list ./internal/... | grep -v -E '(cli|service|binding)$') -v
+#go test $(go list ./internal/... | grep -v -E '(cli|service|binding)$') -v

run_tests.py

@@ -22,6 +22,8 @@ from pathlib import Path
 from typing import List
 import platform
 from enum import Enum
+
+
 class Colors(Enum):
     """ANSI color codes for terminal output"""
 
@@ -29,12 +31,13 @@ class Colors(Enum):
     GREEN = "\033[0;32m"
     YELLOW = "\033[1;33m"
     BLUE = "\033[0;34m"
-    BLACK = '\033[30m'
-    MAGENTA = '\033[35m'
-    CYAN = '\033[36m'
-    WHITE = '\033[37m'
+    BLACK = "\033[30m"
+    MAGENTA = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
     NC = "\033[0m"  # No Color
 
+
 def _is_color_supported() -> bool:
     """
     Detect whether the current environment supports color output
@@ -44,10 +47,10 @@ def _is_color_supported() -> bool:
     Returns:
         result(bool): Whether color output is supported
     """
-     # Non-interactive terminals do not support color output
+    # Non-interactive terminals do not support color output
     if not sys.stdout.isatty():
         return False
-    
+
     # Handle Windows systems
     if sys.platform.startswith("win"):
         try:
@@ -57,10 +60,11 @@ def _is_color_supported() -> bool:
             if not (major >= 10 and build >= 10586):
                 return False
             from ctypes import windll
+
             # Actively enable ANSI support for Windows terminal
             INVALID_HANDLE_VALUE = -1
             kernel32 = windll.kernel32
-            handle = kernel32.GetStdHandle(-11)  # STD_OUTPUT_HANDLE       
+            handle = kernel32.GetStdHandle(-11)  # STD_OUTPUT_HANDLE
             if handle == INVALID_HANDLE_VALUE:
                 return False
             success = kernel32.SetConsoleMode(handle, 7)
@@ -73,21 +77,18 @@ def _is_color_supported() -> bool:
     else:
         try:
             # Detect color support
-            result = subprocess.check_output(
-                ["tput", "colors"], 
-                stderr=subprocess.DEVNULL,
-                text=True
-            )
+            result = subprocess.check_output(["tput", "colors"], stderr=subprocess.DEVNULL, text=True)
             color_count = int(result.strip())
             return color_count >= 8
         # Explicitly catch tput-related exceptions
         except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
-            return False 
-        
+            return False
+
+
 COLOR_SUPPORT = _is_color_supported()
 
-def set_color(s: str,
-              color: str) -> str:
+
+def set_color(s: str, color: str) -> str:
     """
     Wrap input string with specified ANSI terminal color escape sequence.
     Color name argument is case-insensitive. If color is unsupported or invalid,
@@ -103,11 +104,12 @@ def set_color(s: str,
         >>> set_color(s="hello world",color="red")
 
     """
-    
+
     if COLOR_SUPPORT:
         return f"{getattr(Colors, color.strip().upper()).value}{s}{Colors.NC.value}"
     return f"{s}"  # pragma: no cover
 
+
 class TestRunner:
     """RAGFlow Unit Test Runner"""
 
@@ -129,12 +131,12 @@ class TestRunner:
     @staticmethod
     def print_info(message: str) -> None:
         """Print informational message"""
-        print(f"{set_color(s="[INFO]",color="blue")} {message}")
+        print(f"{set_color(s='[INFO]', color='blue')} {message}")
 
     @staticmethod
     def print_error(message: str) -> None:
         """Print error message"""
-        print(f"{set_color(s="[ERROR]",color="red")} {message}")
+        print(f"{set_color(s='[ERROR]', color='red')} {message}")
 
     @staticmethod
     def show_usage() -> None:
@@ -252,20 +254,20 @@ EXAMPLES:
         if self.keyword:
             self.print_info(f"Keyword: {self.keyword}")
 
-        print(f"\n{set_color(s="[EXECUTING]",color='blue')} {' '.join(cmd)}\n")
+        print(f"\n{set_color(s='[EXECUTING]', color='blue')} {' '.join(cmd)}\n")
 
         # Run pytest
         try:
             result = subprocess.run(cmd, check=False)
 
             if result.returncode == 0:
-                print(f"\n{set_color(s="[SUCCESS]",color="green")} All tests passed!")
+                print(f"\n{set_color(s='[SUCCESS]', color='green')} All tests passed!")
 
                 if self.coverage:
                     coverage_dir = self.ut_dir / "htmlcov"
                     if coverage_dir.exists():
                         index_file = coverage_dir / "index.html"
-                        print(f"\n{set_color(s="[INFO]",color="blue")} Coverage report generated:")
+                        print(f"\n{set_color(s='[INFO]', color='blue')} Coverage report generated:")
                         print(f"    {index_file}")
                         print("\nOpen with:")
                         print(f"    - Windows: start {index_file}")
@@ -274,11 +276,11 @@ EXAMPLES:
 
                 return True
             else:
-                print(f"\n{set_color(s="[FAILURE]",color="red")} Some tests failed!")
+                print(f"\n{set_color(s='[FAILURE]', color='red')} Some tests failed!")
                 return False
 
         except KeyboardInterrupt:
-            print(f"\n{set_color(s="[INTERRUPTED]",color="yellow")} Test execution interrupted by user")
+            print(f"\n{set_color(s='[INTERRUPTED]', color='yellow')} Test execution interrupted by user")
             return False
         except Exception as e:
             self.print_error(f"Failed to execute tests: {e}")

test/unit_test/rag/test_naive_dispatch.py

@@ -192,67 +192,60 @@ def naive_module():
             DoclingParser=_ByDocling,
             TCADPParser=_Parser,
         )
-        _stub("deepdoc.parser.figure_parser",
-              VisionFigureParser=_Parser,
-              vision_figure_parser_docx_wrapper_naive=lambda *a, **k: None,
-              vision_figure_parser_pdf_wrapper=lambda *a, **k: None)
-        _stub("deepdoc.parser.pdf_parser",
-              PlainParser=_ByPlaintext,
-              VisionParser=_Parser,
-              RAGFlowPdfParser=_Parser)
+        _stub("deepdoc.parser.figure_parser", VisionFigureParser=_Parser, vision_figure_parser_docx_wrapper_naive=lambda *a, **k: None, vision_figure_parser_pdf_wrapper=lambda *a, **k: None)
+        _stub("deepdoc.parser.pdf_parser", PlainParser=_ByPlaintext, VisionParser=_Parser, RAGFlowPdfParser=_Parser)
         _stub("deepdoc.parser.docling_parser", DoclingParser=_ByDocling)
         _stub("deepdoc.parser.tcadp_parser", TCADPParser=_Parser)
         _stub("deepdoc.parser.utils", extract_pdf_outlines=lambda *a, **k: [])
 
-        _stub("common.parser_config_utils",
-              normalize_layout_recognizer=normalize_layout_recognizer,
-              MINERU_OPTION_KEYS=MINERU_OPTION_KEYS,
-              has_mineru_options=has_mineru_options)
+        _stub("common.parser_config_utils", normalize_layout_recognizer=normalize_layout_recognizer, MINERU_OPTION_KEYS=MINERU_OPTION_KEYS, has_mineru_options=has_mineru_options)
         _stub("common.float_utils", normalize_overlapped_percent=lambda x: x)
         _stub("common.text_utils", normalize_arabic_presentation_forms=lambda x: x)
         _stub("common.token_utils", num_tokens_from_string=lambda s: len((s or "").split()))
 
-        _stub("rag.utils.file_utils",
-              extract_embed_file=lambda *a, **k: None,
-              extract_links_from_pdf=lambda *a, **k: [],
-              extract_links_from_docx=lambda *a, **k: [],
-              extract_html=lambda *a, **k: (None, None))
+        _stub(
+            "rag.utils.file_utils",
+            extract_embed_file=lambda *a, **k: None,
+            extract_links_from_pdf=lambda *a, **k: [],
+            extract_links_from_docx=lambda *a, **k: [],
+            extract_html=lambda *a, **k: (None, None),
+        )
 
-        _stub("rag.nlp",
-              num_tokens_from_string=lambda s: len((s or "").split()),
-              find_codec=lambda b: "utf-8",
-              rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),
-              concat_img=lambda *a, **k: None,
-              naive_merge=lambda *a, **k: [],
-              naive_merge_with_images=lambda *a, **k: [],
-              naive_merge_docx=lambda *a, **k: [],
-              tokenize_chunks=lambda *a, **k: [],
-              doc_tokenize_chunks_with_images=lambda *a, **k: [],
-              tokenize_chunks_with_images=lambda *a, **k: [],
-              tokenize_table=lambda *a, **k: [],
-              append_context2table_image4pdf=lambda *a, **k: [])
+        _stub(
+            "rag.nlp",
+            num_tokens_from_string=lambda s: len((s or "").split()),
+            find_codec=lambda b: "utf-8",
+            rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),
+            concat_img=lambda *a, **k: None,
+            naive_merge=lambda *a, **k: [],
+            naive_merge_with_images=lambda *a, **k: [],
+            naive_merge_docx=lambda *a, **k: [],
+            tokenize_chunks=lambda *a, **k: [],
+            doc_tokenize_chunks_with_images=lambda *a, **k: [],
+            tokenize_chunks_with_images=lambda *a, **k: [],
+            tokenize_table=lambda *a, **k: [],
+            append_context2table_image4pdf=lambda *a, **k: [],
+        )
 
-        _stub("rag.llm.ocr_model",
-              ensure_mineru_from_env=lambda *a, **k: None,
-              get_first_provider_model_name=lambda *a, **k: None)
+        _stub("rag.llm.ocr_model", ensure_mineru_from_env=lambda *a, **k: None, get_first_provider_model_name=lambda *a, **k: None)
 
-        _stub("api.db.joint_services.tenant_model_service",
-              resolve_model_config=lambda *a, **k: None,
-              ensure_mineru_from_env=lambda *a, **k: None,
-              get_tenant_default_model_by_type=lambda *a, **k: None,
-              ensure_opendataloader_from_env=lambda *a, **k: None,
-              ensure_paddleocr_from_env=lambda *a, **k: None,
-              ensure_somark_from_env=lambda *a, **k: None,
-              get_first_provider_model_name=lambda *a, **k: None,
-              get_composite_model_name_by_id=lambda *a, **k: (_ for _ in ()).throw(LookupError()))
+        _stub(
+            "api.db.joint_services.tenant_model_service",
+            resolve_model_config=lambda *a, **k: None,
+            ensure_mineru_from_env=lambda *a, **k: None,
+            get_tenant_default_model_by_type=lambda *a, **k: None,
+            ensure_opendataloader_from_env=lambda *a, **k: None,
+            ensure_paddleocr_from_env=lambda *a, **k: None,
+            ensure_somark_from_env=lambda *a, **k: None,
+            get_first_provider_model_name=lambda *a, **k: None,
+            get_composite_model_name_by_id=lambda *a, **k: (_ for _ in ()).throw(LookupError()),
+        )
 
         _stub("api.db.services.llm_service", LLMBundle=_Parser)
 
         # Stub the docx stack so the file_format checks at import time succeed.
         _stub("docx", Document=_Parser)
-        _stub("docx.opc.pkgreader",
-              _SerializedRelationships=_Parser,
-              _SerializedRelationship=_Parser)
+        _stub("docx.opc.pkgreader", _SerializedRelationships=_Parser, _SerializedRelationship=_Parser)
         _stub("docx.table", Table=_Parser)
         _stub("docx.text.paragraph", Paragraph=_Parser)
         _stub("docx.opc.oxml", parse_xml=lambda *a, **k: None)
@@ -262,9 +255,7 @@ def naive_module():
         # Load naive.py from source — we can do this even with stubs in place
         # because the module only imports the names listed above.
         repo_root = Path(__file__).resolve().parents[3]
-        spec = importlib.util.spec_from_file_location(
-            "rag.app.naive_test_module", repo_root / "rag" / "app" / "naive.py"
-        )
+        spec = importlib.util.spec_from_file_location("rag.app.naive_test_module", repo_root / "rag" / "app" / "naive.py")
         module = importlib.util.module_from_spec(spec)
         sys.modules["rag.app.naive_test_module"] = module
         spec.loader.exec_module(module)
@@ -301,7 +292,9 @@ def test_dispatch_does_not_fall_back_to_mineru_for_plain_text_with_mineru_option
     """layout_recognize='Plain Text' is a known keyword — the dispatcher
     must keep routing to by_plaintext even if mineru_* options are set."""
     parser, name, _lr, _op, _model = _dispatch(
-        naive_module, "Plain Text", {"mineru_lang": "English"},
+        naive_module,
+        "Plain Text",
+        {"mineru_lang": "English"},
     )
     assert name == "plaintext"
     assert parser is _ByPlaintext
@@ -309,7 +302,9 @@ def test_dispatch_does_not_fall_back_to_mineru_for_plain_text_with_mineru_option
 
 def test_dispatch_does_not_fall_back_to_mineru_for_plaintext_with_mineru_options(naive_module):
     parser, name, _lr, _op, _model = _dispatch(
-        naive_module, "plaintext", {"mineru_lang": "English"},
+        naive_module,
+        "plaintext",
+        {"mineru_lang": "English"},
     )
     assert name == "plaintext"
     assert parser is _ByPlaintext
@@ -317,7 +312,9 @@ def test_dispatch_does_not_fall_back_to_mineru_for_plaintext_with_mineru_options
 
 def test_dispatch_does_not_fall_back_to_mineru_for_deepdoc_with_mineru_options(naive_module):
     parser, name, _lr, _op, _model = _dispatch(
-        naive_module, "DeepDOC", {"mineru_lang": "English"},
+        naive_module,
+        "DeepDOC",
+        {"mineru_lang": "English"},
     )
     assert name == "deepdoc"
     assert parser is _ByDeepdoc
@@ -329,7 +326,9 @@ def test_dispatch_falls_back_to_mineru_only_for_unknown_layout_recognize(naive_m
     by_mineru so the operator's MinerU intent is honored (issue #17114)."""
     stale_uuid = "06d85f8e819111f1995ef33d60f3a479"
     parser, name, _lr, _op, _model = _dispatch(
-        naive_module, stale_uuid, {"mineru_lang": "English"},
+        naive_module,
+        stale_uuid,
+        {"mineru_lang": "English"},
     )
     assert name == "mineru"
     # The fallback branch reassigns parser=by_mineru directly (not via

web/src/locales/fr.ts

@@ -1031,7 +1031,7 @@ Applicable lorsque vous avez besoin que le LLM résume le document entier.
       embedModalTitle: 'Intégrer dans une page web',
       embedUserIdPlaceholder: 'ex. user-001',
       embedUserIdTooltip:
-        'Chaîne de texte (255 caractères maximum) identifiant l\'utilisateur final de la page intégrée. Elle est ajoutée à l\'URL d\'intégration comme paramètre userId.',
+        "Chaîne de texte (255 caractères maximum) identifiant l'utilisateur final de la page intégrée. Elle est ajoutée à l'URL d'intégration comme paramètre userId.",
       comingSoon: 'Bientôt disponible',
       fullScreenTitle: 'Intégration complète',
       fullScreenDescription:
@@ -1504,7 +1504,8 @@ Exemple : Virtual Hosted Style`,
         'Connectez un site SharePoint via Microsoft Graph pour synchroniser ses bibliothèques de documents.',
       sharepointSiteUrlTip:
         'URL complète du site SharePoint à indexer, ex. : https://contoso.sharepoint.com/sites/MonSite. Nécessite une application Azure AD avec les permissions applicatives Sites.Read.All et Files.Read.All (consentement administrateur).',
-      azure_devopsDescription: 'Connectez Azure DevOps pour synchroniser les fichiers du dépôt et les pull requests.',
+      azure_devopsDescription:
+        'Connectez Azure DevOps pour synchroniser les fichiers du dépôt et les pull requests.',
       bitbucketDescription:
         'Connectez Bitbucket pour synchroniser le contenu des PR.',
       bitbucketTopWorkspaceTip:

web/src/locales/it.ts

@@ -1093,7 +1093,7 @@ Questa funzionalità di auto-tagging migliora il recupero aggiungendo un ulterio
       embedModalTitle: 'Incorpora nella pagina web',
       embedUserIdPlaceholder: 'es. user-001',
       embedUserIdTooltip:
-        'Stringa di testo (massimo 255 caratteri) che identifica l\'utente finale della pagina incorporata. Viene aggiunta all\'URL di incorporamento come parametro userId.',
+        "Stringa di testo (massimo 255 caratteri) che identifica l'utente finale della pagina incorporata. Viene aggiunta all'URL di incorporamento come parametro userId.",
       published: 'Pubblicato',
       publishedTooltip:
         "Usa la versione pubblicata per questo embed. Quando abilitato, l'URL generato include release=true.",
@@ -1283,7 +1283,8 @@ Esempio: Virtual Hosted Style`,
         'Connetti un sito SharePoint tramite Microsoft Graph per sincronizzare le sue librerie documenti.',
       sharepointSiteUrlTip:
         "URL completo del sito SharePoint da indicizzare, es. https://contoso.sharepoint.com/sites/MySite. Richiede un'app Azure AD con permessi applicativi Sites.Read.All e Files.Read.All (consenso admin).",
-      azure_devopsDescription: 'Collega Azure DevOps per sincronizzare i file del repository e le pull request.',
+      azure_devopsDescription:
+        'Collega Azure DevOps per sincronizzare i file del repository e le pull request.',
       bitbucketDescription:
         'Connetti Bitbucket per sincronizzare il contenuto delle PR.',
       bitbucketTopWorkspaceTip:

web/src/locales/ja.ts

@@ -1338,7 +1338,8 @@ export default {
         'Microsoft Graph経由でSharePointサイトに接続し、ドキュメントライブラリを同期します。',
       sharepointSiteUrlTip:
         'インデックス対象のSharePointサイトの完全なURLです（例: https://contoso.sharepoint.com/sites/MySite）。Sites.Read.AllおよびFiles.Read.Allのアプリケーション権限（管理者の同意）を持つAzure ADアプリが必要です。',
-      azure_devopsDescription: 'Azure DevOps に接続し、リポジトリのファイルとプルリクエストを同期します。',
+      azure_devopsDescription:
+        'Azure DevOps に接続し、リポジトリのファイルとプルリクエストを同期します。',
       bitbucketDescription: 'Bitbucketに接続し、PRの内容を同期します。',
       bitbucketTopWorkspaceTip:
         'インデックス対象のBitbucketワークスペースです（例: https://bitbucket.org/atlassian/workspace の「atlassian」）。',

web/src/locales/ko.ts

@@ -1255,7 +1255,8 @@ export default {
         'Microsoft Graph를 통해 SharePoint 사이트를 연결하여 문서 라이브러리를 동기화합니다.',
       sharepointSiteUrlTip:
         '인덱싱할 SharePoint 사이트의 전체 URL (예: https://contoso.sharepoint.com/sites/MySite). Sites.Read.All 및 Files.Read.All 애플리케이션 권한이 있는 Azure AD 앱이 필요합니다 (관리자 동의).',
-      azure_devopsDescription: 'Azure DevOps를 연결하여 리포지토리 파일과 풀 리퀘스트를 동기화합니다.',
+      azure_devopsDescription:
+        'Azure DevOps를 연결하여 리포지토리 파일과 풀 리퀘스트를 동기화합니다.',
       bitbucketDescription: 'Bitbucket을 연결하여 PR 콘텐츠를 동기화합니다.',
       bitbucketTopWorkspaceTip:
         '인덱싱할 Bitbucket 워크스페이스 (예: https://bitbucket.org/atlassian/workspace 의 "atlassian").',

web/src/locales/ru.ts

@@ -1066,7 +1066,8 @@ export default {
         'Загрузите OAuth JSON, сгенерированный в Google Console. Если он содержит только учетные данные клиента, выполните одноразовое подтверждение в браузере, чтобы получить долгоживущие токены обновления.',
       dropboxDescription:
         'Подключите ваш Dropbox для синхронизации файлов и папок из выбранного аккаунта.',
-      azure_devopsDescription: "Подключите Azure DevOps для синхронизации файлов репозитория и pull request'ов.",
+      azure_devopsDescription:
+        "Подключите Azure DevOps для синхронизации файлов репозитория и pull request'ов.",
       bitbucketDescription:
         'Подключите Bitbucket для синхронизации содержимого PR.',
       bitbucketTopWorkspaceTip:
@@ -1310,8 +1311,7 @@ export default {
       baseUrlNameMessage: 'Пожалуйста, введите ваш базовый URL',
       mwsApiUrl: 'URL API',
       mwsApiUrlMessage: 'Введите URL API проекта MWS',
-      mwsApiUrlPlaceholder:
-        'https://gpt.mwsapis.ru/projects/<project>',
+      mwsApiUrlPlaceholder: 'https://gpt.mwsapis.ru/projects/<project>',
       mwsToken: 'Токен',
       mwsTokenMessage: 'Введите токен MWS',
       mwsTokenPlaceholder: 'API-ключ сервисного аккаунта MWS',

web/src/locales/tr.ts

@@ -1054,7 +1054,7 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
       embedModalTitle: 'Web sayfasına göm',
       embedUserIdPlaceholder: 'örn. user-001',
       embedUserIdTooltip:
-        'Gömülü sayfanın son kullanıcısını tanımlayan bir metin (en fazla 255 karakter). Gömme URL\'sine userId parametresi olarak eklenir.',
+        "Gömülü sayfanın son kullanıcısını tanımlayan bir metin (en fazla 255 karakter). Gömme URL'sine userId parametresi olarak eklenir.",
       published: 'Yayınlandı',
       publishedTooltip: 'Bu göm için yayınlanan sürümü kullanın.',
       embedType: 'Embed türü',
@@ -1224,7 +1224,8 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
         "Google Console'dan oluşturulan OAuth JSON'ını yükleyin. Yalnızca istemci kimlik bilgilerini içeriyorsa, uzun ömürlü yenileme tokenları oluşturmak için tarayıcı tabanlı doğrulamayı bir kez çalıştırın.",
       dropboxDescription:
         "Seçilen bir hesaptan dosya ve klasörleri senkronize etmek için Dropbox'ınızı bağlayın.",
-      azure_devopsDescription: "Depo dosyalarını ve pull request'leri senkronize etmek için Azure DevOps'u bağlayın.",
+      azure_devopsDescription:
+        "Depo dosyalarını ve pull request'leri senkronize etmek için Azure DevOps'u bağlayın.",
       bitbucketDescription:
         "PR içeriğini senkronize etmek için Bitbucket'ı bağlayın.",
       bitbucketTopWorkspaceTip:

web/src/pages/agent/hooks/use-save-graph.test.ts

@@ -28,9 +28,9 @@ describe('shouldAutosaveCanvas', () => {
   });
 
   it('skips an empty nodes/edges store so autosave cannot wipe a pipeline', () => {
-    expect(
-      shouldAutosaveCanvas({ ...ready, nodeCount: 0, edgeCount: 0 }),
-    ).toBe(false);
+    expect(shouldAutosaveCanvas({ ...ready, nodeCount: 0, edgeCount: 0 })).toBe(
+      false,
+    );
   });
 
   it('still autosaves when there are nodes but no edges', () => {