fix(rag): unify parser_config delimiter defaults on a single constant (#19058)

api/db/services/file_service.py

@@ -43,6 +43,7 @@ from common.constants import MAXIMUM_PAGE_NUMBER, FileSource, ParserType, TaskSt
 from common.misc_utils import get_uuid
 from common.ssrf_guard import assert_url_is_safe
 from rag.llm.cv_model import GptV4
+from rag.nlp.delim import DEFAULT_DELIMITER
 
 
 class FileService(CommonService):
@@ -710,7 +711,7 @@ class FileService(CommonService):
             pass
 
         FACTORY = {ParserType.PRESENTATION.value: presentation, ParserType.PICTURE.value: picture, ParserType.AUDIO.value: audio, ParserType.EMAIL.value: email}
-        parser_config = {"chunk_token_num": 16096, "delimiter": "\n!?;。；！？", "layout_recognize": layout_recognize or "Plain Text"}
+        parser_config = {"chunk_token_num": 16096, "delimiter": DEFAULT_DELIMITER, "layout_recognize": layout_recognize or "Plain Text"}
         kwargs = {"lang": "English", "callback": dummy, "parser_config": parser_config, "from_page": 0, "to_page": MAXIMUM_PAGE_NUMBER, "tenant_id": current_user.id if current_user else tenant_id}
         file_type = filename_type(filename)
         if img_base64 and file_type == FileType.VISUAL.value:

deepdoc/parser/txt_parser.py

@@ -20,6 +20,7 @@ import re
 from deepdoc.parser.utils import get_text
 from rag.nlp import MergeStrategy, merge_paragraphs
 from rag.nlp.delim import (
+    DEFAULT_DELIMITER,
     compile_delimiter_pattern,
     normalize_text_newlines,
     parse_delimiter_field,
@@ -27,12 +28,12 @@ from rag.nlp.delim import (
 
 
 class RAGFlowTxtParser:
-    def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
+    def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):
         txt = get_text(fnm, binary)
         return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)
 
     @classmethod
-    def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False):
+    def parser_txt(cls, txt, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):
         if not isinstance(txt, str):
             raise TypeError("txt type should be str!")
         txt = normalize_text_newlines(txt)

internal/ingestion/component/chunker/sentence_boundary.go

@@ -26,7 +26,7 @@ import "regexp"
 //
 // The token chunker's TEXT path (naive_merge port) uses a different delimiter
 // — sentenceDelimiter in token.go, mirroring Python naive_merge's production
-// delimiter "\n!?。；！？" (no ". "). The two constants mirror two distinct
-// Python constants and differ only in that English boundary, so they are
-// intentionally NOT merged into one.
+// default DEFAULT_DELIMITER ("\n!?;。；！？", no ". "). The two constants mirror
+// two distinct Python constants and differ only in that English boundary, so
+// they are intentionally NOT merged into one.
 var sentenceBoundaryRe = regexp.MustCompile(`([。!?？；！\n]|\. )`)

internal/ingestion/component/chunker/sentence_boundary_test.go

@@ -39,11 +39,12 @@ func TestSentenceBoundary_SharedRegexMatchesPython(t *testing.T) {
 
 // TestSentenceBoundary_TokenTextPathDelimiterDistinct pins sentenceDelimiter
 // (the token-chunker TEXT path / naive_merge port) to Python naive_merge's
-// production delimiter "\n!?。；！？" — deliberately WITHOUT the English ". "
-// boundary. The two constants mirror two distinct Python delimiters; asserting
-// the exact difference documents that this is intentional, not a drift.
+// production default DEFAULT_DELIMITER ("\n!?;。；！？", rag/nlp/delim.py) —
+// deliberately WITHOUT the English ". " boundary. The two constants mirror two
+// distinct Python delimiters; asserting the exact difference documents that
+// this is intentional, not a drift.
 func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {
-	const want = `(\n|[!?。；！？])`
+	const want = `(\n|[!?;。；！？])`
 	if got := sentenceDelimiter.String(); got != want {
 		t.Errorf("sentenceDelimiter = %q, want %q (Python naive_merge delimiter)", got, want)
 	}
@@ -52,6 +53,11 @@ func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {
 	if sentenceDelimiter.MatchString("Hello. World") {
 		t.Error("sentenceDelimiter split on '. ' but naive_merge's delimiter has no '. ' boundary")
 	}
+	// The text-path delimiter must split on the ASCII semicolon, matching the
+	// unified Python default (issue #18562: the "; " omission was drift).
+	if !sentenceDelimiter.MatchString("one; two") {
+		t.Error("sentenceDelimiter should split on ';': naive_merge's default includes ';'")
+	}
 	if !sentenceBoundaryRe.MatchString("Hello. World") {
 		t.Error("sentenceBoundaryRe should split on '. '")
 	}

internal/ingestion/component/chunker/token.go

@@ -364,14 +364,15 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child
 
 // sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the
 // naive_merge port). It mirrors the delimiter Python's naive_merge receives in
-// production: rag/app/naive.py passes "\n!?。；！？", which includes ASCII "!"
-// and "?" plus the CJK punctuation "。；！？" but NOT an English ". " boundary.
+// production: rag/app/naive.py passes DEFAULT_DELIMITER ("\n!?;。；！？", defined
+// once in rag/nlp/delim.py), which includes ASCII "!", "?" and ";" plus the CJK
+// punctuation "。；！？" but NOT an English ". " boundary.
 //
 // The Title chunker and the token chunker's JSON path use the shared
 // sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,
 // which adds ". "). The two constants mirror two distinct Python delimiters and
 // differ only in that English boundary, exactly as Python does.
-var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。；！？])`)
+var sentenceDelimiter = regexp.MustCompile(`(\n|[!?;。；！？])`)
 
 // overlapCut returns the visible-text rune offset where the overlap prefix
 // begins, mirroring the cut computed inside computeOverlapPrefix. It is split

rag/app/book.py

@@ -67,6 +67,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
     Since a book is long and not all the parts are useful, if it's a PDF,
     please set up the page ranges for every book in order eliminate negative effects and save elapsed computing time.
     """
+    # Deliberate carve-out from DEFAULT_DELIMITER (#18562): "book" targets
+    # Chinese prose, so this kwargs dict and the naive_merge fallback below
+    # keep their own Chinese-only delimiter sets, not the unified constant.
     parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
     doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
     doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])

rag/app/email.py

@@ -20,7 +20,7 @@ from email.parser import BytesParser
 from rag.app.naive import chunk as naive_chunk
 from common.constants import MAXIMUM_PAGE_NUMBER
 import re
-from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks
+from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks, DEFAULT_DELIMITER
 from deepdoc.parser import HtmlParser, TxtParser
 from timeit import default_timer as timer
 import io
@@ -41,7 +41,7 @@ def chunk(
     eng = lang.lower() == "english"  # is_english(cks)
     parser_config = kwargs.get(
         "parser_config",
-        {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"},
+        {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"},
     )
     doc = {
         "docnm_kwd": filename,
@@ -99,7 +99,7 @@ def chunk(
     chunks = naive_merge(
         sections,
         int(parser_config.get("chunk_token_num", 128)),
-        parser_config.get("delimiter", "\n!?。；！？"),
+        parser_config.get("delimiter", DEFAULT_DELIMITER),
     )
 
     main_res.extend(tokenize_chunks(chunks, doc, eng, None, language=lang))

rag/app/laws.py

@@ -24,7 +24,7 @@ from docx.text.paragraph import Paragraph
 
 from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
 from deepdoc.parser.utils import get_text
-from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge
+from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge, DEFAULT_DELIMITER
 from rag.nlp import rag_tokenizer, Node
 from deepdoc.parser import PdfParser, DocxParser, HtmlParser
 from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
@@ -169,7 +169,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
     """
     Supported file formats are docx, pdf, txt.
     """
-    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
+    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
     doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
     doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])
     pdf_parser = None

rag/app/manual.py

@@ -21,7 +21,7 @@ import re
 from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
 from io import BytesIO
 from deepdoc.parser.utils import extract_pdf_outlines
-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img
+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img, DEFAULT_DELIMITER
 from common.token_utils import num_tokens_from_string
 from deepdoc.parser import PdfParser, DocxParser
 from deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper, vision_figure_parser_docx_wrapper
@@ -139,7 +139,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
     """
     Only pdf is supported.
     """
-    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
+    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
     pdf_parser = None
     doc = {"docnm_kwd": filename}
     doc["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", doc["docnm_kwd"]))

rag/app/naive.py

@@ -52,6 +52,7 @@ from common.parser_config_utils import has_mineru_options, normalize_layout_reco
 from common.text_utils import normalize_arabic_presentation_forms
 from rag.nlp import (
     concat_img,
+    DEFAULT_DELIMITER,
     find_codec,
     naive_merge,
     naive_merge_with_images,
@@ -1067,7 +1068,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
 
     lang = lang or "Chinese"
     is_english = lang.lower() == "english"  # is_english(cks)
-    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC", "analyze_hyperlink": True})
+    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC", "analyze_hyperlink": True})
 
     child_deli = (parser_config.get("children_delimiter") or "").encode("utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")
     cust_child_deli = re.findall(r"`([^`]+)`", child_deli)
@@ -1133,7 +1134,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
 
         # chunks list[dict]
         # images list - index of image chunk in chunks
-        chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), table_context_size, image_context_size)
+        chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), table_context_size, image_context_size)
 
         vision_figure_parser_docx_wrapper_naive(
             chunks=chunks,
@@ -1261,7 +1262,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
 
     elif re.search(r"\.(txt|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|sql)$", filename, re.IGNORECASE):
         callback(0.1, "Start to parse.")
-        sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", "\n!?;。；！？"))
+        sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", DEFAULT_DELIMITER))
         sections = _normalize_section_text_for_rtl_presentation_forms(sections)
         logging.info("TxtParser produced %d sections for %s", len(sections), filename)
         callback(0.8, "Finish parsing.")
@@ -1273,7 +1274,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
             filename,
             binary,
             separate_tables=False,
-            delimiter=parser_config.get("delimiter", "\n!?;。；！？"),
+            delimiter=parser_config.get("delimiter", DEFAULT_DELIMITER),
             return_section_images=True,
         )
         sections = _normalize_section_text_for_rtl_presentation_forms(sections)
@@ -1428,10 +1429,10 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
                 section_images = None
 
         if section_images:
-            chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent)
+            chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)
             res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli, language=lang))
         else:
-            chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent)
+            chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)
 
             res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang))
 

rag/app/one.py

@@ -20,7 +20,7 @@ import re
 
 from deepdoc.parser.utils import get_text
 from rag.app import naive
-from rag.nlp import rag_tokenizer, tokenize
+from rag.nlp import rag_tokenizer, tokenize, DEFAULT_DELIMITER
 from deepdoc.parser import PdfParser, ExcelParser, HtmlParser
 from deepdoc.parser.figure_parser import vision_figure_parser_docx_wrapper_naive
 from rag.app.naive import by_plaintext, PARSERS
@@ -65,7 +65,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
     Supported file formats are docx, pdf, excel, txt.
     One file forms a chunk which maintains original text order.
     """
-    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
+    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
     eng = lang.lower() == "english"  # is_english(cks)
 
     if re.search(r"\.docx$", filename, re.IGNORECASE):

rag/app/paper.py

@@ -20,7 +20,7 @@ import re
 
 from deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper
 from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context
+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context, DEFAULT_DELIMITER
 from deepdoc.parser import PdfParser
 import numpy as np
 from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
@@ -137,7 +137,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
     Only pdf is supported.
     The abstract of the paper will be sliced as an entire chunk, and will not be sliced partly.
     """
-    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。；！？", "layout_recognize": "DeepDOC"})
+    parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
     if re.search(r"\.pdf$", filename, re.IGNORECASE):
         layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
         tenant_id = kwargs.get("tenant_id")

rag/flow/parser/parser.py

@@ -38,6 +38,7 @@ from api.db.joint_services.tenant_model_service import (
 )
 from api.db.services.tenant_model_instance_service import TenantModelInstanceService
 from api.db.services.tenant_model_provider_service import TenantModelProviderService
+from rag.nlp.delim import DEFAULT_DELIMITER
 from api.db.services.tenant_model_service import TenantModelService
 from common import settings
 from common.constants import LLMType
@@ -1148,7 +1149,7 @@ class Parser(ProcessBase):
             name,
             blob,
             conf.get("chunk_token_num", 128),
-            conf.get("delimiter", "\n!?;。；！？"),
+            conf.get("delimiter", DEFAULT_DELIMITER),
             keep_delimiters=True,
         )
         if conf.get("output_format") == "json":

rag/nlp/__init__.py

@@ -32,6 +32,7 @@ from common.token_utils import num_tokens_from_string
 # Re-exported below for backwards compatibility; the canonical parser lives
 # in ``rag.nlp.delim``.
 from rag.nlp.delim import (
+    DEFAULT_DELIMITER,
     compile_delimiter_pattern,
     has_wrapped_delimiter,
     normalize_text_newlines,
@@ -1415,7 +1416,7 @@ def _apply_overlap_unconditional(chunks, overlapped_percent):
     return out
 
 
-def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。；！？", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
+def naive_merge(sections: str | list, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
     """Split sections into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""
     if not sections:
         return []
@@ -1478,7 +1479,7 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。；
     return _apply_overlap_unconditional(cks, overlapped_percent)
 
 
-def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。；！？", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
+def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
     """Split texts (with images) into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""
     if not texts or len(texts) != len(images):
         return [], []
@@ -1834,7 +1835,7 @@ def _merge_cks(cks, chunk_token_num, has_custom):
 def naive_merge_docx(
     sections,
     chunk_token_num=128,
-    delimiter="\n。；！？",
+    delimiter=DEFAULT_DELIMITER,
     table_context_size=0,
     image_context_size=0,
 ):

rag/nlp/delim.py

@@ -78,6 +78,15 @@ import re
 # Match a backtick-wrapped token. Case-sensitive on purpose (see #17384).
 _BACKTICK_RE = re.compile(r"`([^`]+)`")
 
+# The single source of truth for the default value of the
+# ``parser_config.delimiter`` field (issue #18562). Before this constant
+# existed, the per-callback ``parser_config.get("delimiter", ...)`` defaults
+# drifted across several copy-pasted values: txt/markdown shipped this full
+# 8-character set, docx/image/email omitted the ASCII ``;``, and
+# ``rag/app/book.py`` keeps its own deliberate Chinese-only set. Every backend
+# default must reference this constant so the value can never drift again.
+DEFAULT_DELIMITER = "\n!?;。；！？"
+
 
 def normalize_text_newlines(text: str) -> str:
     """Normalize CRLF and standalone CR to LF in source text."""

rag/svr/task_executor.py

@@ -82,7 +82,7 @@ from api.db.joint_services.tenant_model_service import get_tenant_default_model_
 from common.versions import get_ragflow_version
 from api.db.db_models import close_connection
 from rag.app import laws, paper, presentation, manual, qa, table, book, resume, picture, naive, one, audio, email, tag
-from rag.nlp import search, rag_tokenizer, add_positions
+from rag.nlp import search, rag_tokenizer, add_positions, DEFAULT_DELIMITER
 
 from common.token_utils import num_tokens_from_string, truncate
 from rag.utils.redis_conn import REDIS_CONN, RedisDistributedLock
@@ -342,7 +342,7 @@ async def build_chunks(task, progress_callback, on_chunking_start=None):
         "parser_id": task["parser_id"],
         "chunk_token_num": parser_config_for_chunk.get("chunk_token_num", 128),
         "overlapped_percent": normalize_overlapped_percent(parser_config_for_chunk.get("overlapped_percent", 0)),
-        "delimiter": parser_config_for_chunk.get("delimiter", "\n!?。；！？"),
+        "delimiter": parser_config_for_chunk.get("delimiter", DEFAULT_DELIMITER),
         "from_page": task["from_page"],
         "to_page": task["to_page"],
         "language": task["language"],

rag/svr/task_executor_refactor/chunk_service.py

@@ -40,7 +40,7 @@ from common.misc_utils import thread_pool_exec
 from common.float_utils import normalize_overlapped_percent
 from api.db.services.document_service import DocumentService
 from api.db.services.task_service import TaskService
-from rag.nlp import search
+from rag.nlp import search, DEFAULT_DELIMITER
 from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID
 from rag.svr.task_executor_refactor.task_context import TaskContext
 from rag.utils.base64_image import image2id
@@ -171,7 +171,7 @@ class ChunkService:
             "parser_id": ctx.parser_id,
             "chunk_token_num": ctx.parser_config.get("chunk_token_num", 128),
             "overlapped_percent": normalize_overlapped_percent(ctx.parser_config.get("overlapped_percent", 0)),
-            "delimiter": ctx.parser_config.get("delimiter", "\n!?。；！？"),
+            "delimiter": ctx.parser_config.get("delimiter", DEFAULT_DELIMITER),
             "from_page": ctx.from_page,
             "to_page": ctx.to_page,
             "language": ctx.language,

test/unit_test/rag/nlp/test_default_delimiter_constant.py

@@ -0,0 +1,151 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+"""Pin the single source of truth for the default ``parser_config.delimiter``.
+
+Before #18562 the backend carried four distinct copy-pasted defaults for the
+same field: txt/markdown merged with ``"\\n!?;。；！？"``, docx/image/email with
+``"\\n!?。；！？"`` (no ASCII ``;``), the ``naive_merge*`` signatures with the
+Chinese-only ``"\\n。；！？"``, and the UI/API layer with ``"\\n"``. The
+docx/image/email ``;`` omission was user-visible: English semicolon prose
+chunked differently (often one oversized chunk) than the same text as
+.txt/.md.
+
+These tests pin the unified state:
+
+* ``rag.nlp.delim.DEFAULT_DELIMITER`` is the 8-character set and parses to
+  exactly those delimiters.
+* Every backend ``parser_config.get("delimiter", ...)`` fallback, kwargs
+  default dict, and ``naive_merge*`` / ``RAGFlowTxtParser`` signature default
+  references the constant (AST-checked, so a future copy-pasted literal fails
+  loudly instead of drifting silently).
+* ``rag/app/book.py`` deliberately keeps its own Chinese-only defaults (the
+  kwargs default dict and the ``naive_merge`` fallback) — a product decision,
+  not drift, pinned here as a carve-out.
+* Behaviourally, semicolon-separated English text splits at ``;`` under the
+  unified default but stays whole under the pre-fix docx default.
+"""
+
+import ast
+import inspect
+from pathlib import Path
+
+from rag.nlp import naive_merge, naive_merge_docx, naive_merge_with_images
+from rag.nlp.delim import DEFAULT_DELIMITER, parse_delimiter_field
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+
+# Production files whose ``delimiter`` defaults must reference the constant,
+# mapped to the exact number of defaults each file carries. Exact per-file
+# counts (rather than an aggregate floor) make a deleted fallback or a newly
+# copy-pasted literal fail loudly instead of silently re-drifting.
+EXPECTED_DELIMITER_DEFAULTS = {
+    "rag/app/naive.py": 6,
+    "rag/app/email.py": 2,
+    "rag/app/laws.py": 1,
+    "rag/app/one.py": 1,
+    "rag/app/manual.py": 1,
+    "rag/app/paper.py": 1,
+    "rag/svr/task_executor.py": 1,
+    "rag/svr/task_executor_refactor/chunk_service.py": 1,
+    "api/db/services/file_service.py": 1,
+    "rag/flow/parser/parser.py": 1,
+    "deepdoc/parser/txt_parser.py": 2,
+    "rag/nlp/__init__.py": 3,
+}
+
+
+def test_constant_value():
+    assert DEFAULT_DELIMITER == "\n!?;。；！？"
+    # Newline + ASCII !, ?, ; + CJK 。；！？ — 8 delimiters, deduplicated.
+    assert parse_delimiter_field(DEFAULT_DELIMITER) == ["\n", "!", "?", ";", "。", "；", "！", "？"]
+
+
+def _delimiter_defaults(tree):
+    """Yield (kind, default_node) for every ``delimiter`` default in the module."""
+    for node in ast.walk(tree):
+        if (
+            isinstance(node, ast.Call)
+            and isinstance(node.func, ast.Attribute)
+            and node.func.attr == "get"
+            and node.args
+            and isinstance(node.args[0], ast.Constant)
+            and node.args[0].value == "delimiter"
+            and len(node.args) > 1
+            and isinstance(node.args[1], (ast.Constant, ast.Name))
+        ):
+            yield ".get", node.args[1]
+        if isinstance(node, ast.Dict):
+            for key, value in zip(node.keys, node.values):
+                if isinstance(key, ast.Constant) and key.value == "delimiter" and isinstance(value, (ast.Constant, ast.Name)):
+                    yield "dict", value
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            pairs = list(zip(node.args.args[-len(node.args.defaults) :], node.args.defaults)) if node.args.defaults else []
+            pairs += [(a, a.default) for a in node.args.kwonlyargs if a.default is not None]
+            for arg, default in pairs:
+                if arg.arg == "delimiter" and not (isinstance(default, ast.Constant) and default.value is None):
+                    yield f"sig:{node.name}", default
+
+
+def test_all_backend_defaults_reference_the_constant():
+    for relpath, expected in EXPECTED_DELIMITER_DEFAULTS.items():
+        tree = ast.parse((REPO_ROOT / relpath).read_text(encoding="utf-8"))
+        defaults = list(_delimiter_defaults(tree))
+        assert len(defaults) == expected, (
+            f"{relpath}: expected exactly {expected} delimiter defaults, found {len(defaults)}; update EXPECTED_DELIMITER_DEFAULTS in the same change so a removed or new default stays loud"
+        )
+        for kind, default in defaults:
+            assert isinstance(default, ast.Name) and default.id == "DEFAULT_DELIMITER", f"{relpath}: '{kind}' delimiter default is {ast.dump(default)}, expected the DEFAULT_DELIMITER constant"
+
+
+def test_book_keeps_its_deliberate_chinese_only_default():
+    """``book`` chunking targets Chinese prose; its defaults are a product choice."""
+    tree = ast.parse((REPO_ROOT / "rag/app/book.py").read_text(encoding="utf-8"))
+    found = list(_delimiter_defaults(tree))
+    kwargs_dict_defaults = [d for kind, d in found if kind == "dict"]
+    merge_defaults = [d for kind, d in found if kind == ".get"]
+    # Two distinct Chinese-only sets, both deliberate carve-outs from
+    # DEFAULT_DELIMITER: the chunk() kwargs default dict ("\n!?。；！？")
+    # and the naive_merge fallback ("\n。；！？").
+    assert any(isinstance(d, ast.Constant) and d.value == "\n!?。；！？" for d in kwargs_dict_defaults)
+    assert any(isinstance(d, ast.Constant) and d.value == "\n。；！？" for d in merge_defaults)
+
+
+def test_merge_signatures_carry_the_constant():
+    for func in (naive_merge, naive_merge_with_images, naive_merge_docx):
+        default = inspect.signature(func).parameters["delimiter"].default
+        assert default == DEFAULT_DELIMITER, f"{func.__name__} signature default diverges from DEFAULT_DELIMITER"
+
+
+def test_semicolon_english_text_splits_under_the_unified_default():
+    """The reported symptom: ``;`` clauses must split like they do for txt/md."""
+    text = "; ".join(
+        [
+            "The rain fell gently over the quiet town",
+            "the sun broke through the grey clouds",
+            "the wind stirred the leaves in the yard",
+            "children ran along the wet stone path",
+            "a distant bell rang twice before noon",
+        ]
+    )
+    sections = [(text, "")]
+    unified = naive_merge(sections, chunk_token_num=32, delimiter=DEFAULT_DELIMITER)
+    assert len(unified) > 1, "semicolon-separated clauses should not land in a single chunk"
+
+    # Pre-fix docx/image/email default (no ASCII ';'): same text, one blob.
+    pre_fix_docx_default = "\n!?。；！？"
+    legacy = naive_merge(sections, chunk_token_num=32, delimiter=pre_fix_docx_default)
+    assert len(legacy) < len(unified), "the unified default must split semicolon prose more than the ';'less drift default"

test/unit_test/rag/test_naive_dispatch.py

@@ -213,6 +213,9 @@ def naive_module():
 
         _stub(
             "rag.nlp",
+            # naive.py imports the unified delimiter constant (#18562); the
+            # dispatch tests never read the value — only the name must exist.
+            DEFAULT_DELIMITER="\n!?;。；！？",
             num_tokens_from_string=lambda s: len((s or "").split()),
             find_codec=lambda b: "utf-8",
             rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),