Coverage for haystack/components/preprocessors/document_preprocessor.py: 96%
47 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-12 09:42 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5from collections.abc import Callable
6from typing import TYPE_CHECKING, Any, Literal
8from haystack import Document, Pipeline, default_from_dict, default_to_dict, super_component
9from haystack.components.preprocessors.document_cleaner import DocumentCleaner
10from haystack.components.preprocessors.document_splitter import DocumentSplitter, Language
11from haystack.utils import deserialize_callable, serialize_callable
14@super_component
15class DocumentPreprocessor:
16 """
17 A SuperComponent that first splits and then cleans documents.
19 This component consists of a DocumentSplitter followed by a DocumentCleaner in a single pipeline.
20 It takes a list of documents as input and returns a processed list of documents.
22 Usage example:
23 ```python
24 from haystack import Document
25 from haystack.components.preprocessors import DocumentPreprocessor
27 doc = Document(content="I love pizza!")
28 preprocessor = DocumentPreprocessor()
29 result = preprocessor.run(documents=[doc])
30 print(result["documents"])
31 ```
32 """
34 def __init__( # noqa: PLR0913 (too-many-arguments)
35 self,
36 *,
37 # --- DocumentSplitter arguments ---
38 split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence", "token"] = "word",
39 split_length: int = 250,
40 split_overlap: int = 0,
41 split_threshold: int = 0,
42 splitting_function: Callable[[str], list[str]] | None = None,
43 respect_sentence_boundary: bool = False,
44 language: Language = "en",
45 use_split_rules: bool = True,
46 extend_abbreviations: bool = True,
47 tokenizer_encoding: str = "o200k_base",
48 # --- DocumentCleaner arguments ---
49 remove_empty_lines: bool = True,
50 remove_extra_whitespaces: bool = True,
51 remove_repeated_substrings: bool = False,
52 keep_id: bool = False,
53 remove_substrings: list[str] | None = None,
54 remove_regex: str | None = None,
55 unicode_normalization: Literal["NFC", "NFKC", "NFD", "NFKD"] | None = None,
56 ascii_only: bool = False,
57 ) -> None:
58 """
59 Initialize a DocumentPreProcessor that first splits and then cleans documents.
61 **Splitter Parameters**:
62 :param split_by: The unit of splitting: "function", "page", "passage", "period", "word", "line",
63 "sentence", or "token".
64 :param split_length: The maximum number of units (words, lines, pages, and so on) in each split.
65 :param split_overlap: The number of overlapping units between consecutive splits.
66 :param split_threshold: The minimum number of units per split. If a split is smaller than this, it's merged
67 with the previous split.
68 :param splitting_function: A custom function for splitting if `split_by="function"`.
69 :param respect_sentence_boundary: If `True`, splits by words but tries not to break inside a sentence.
70 :param language: Language used by the sentence tokenizer if `split_by="sentence"` or
71 `respect_sentence_boundary=True`.
72 :param use_split_rules: Whether to apply additional splitting heuristics for the sentence splitter.
73 :param extend_abbreviations: Whether to extend the sentence splitter with curated abbreviations for certain
74 languages.
75 :param tokenizer_encoding: The tiktoken encoding to use when `split_by="token"`. Defaults to
76 `"o200k_base"` (current OpenAI models). Only used when `split_by="token"`.
78 **Cleaner Parameters**:
79 :param remove_empty_lines: If `True`, removes empty lines.
80 :param remove_extra_whitespaces: If `True`, removes extra whitespaces.
81 :param remove_repeated_substrings: If `True`, removes repeated substrings like headers/footers across pages.
82 :param keep_id: If `True`, keeps the original document IDs.
83 :param remove_substrings: A list of strings to remove from the document content.
84 :param remove_regex: A regex pattern whose matches will be removed from the document content.
85 :param unicode_normalization: Unicode normalization form to apply to the text, for example `"NFC"`.
86 :param ascii_only: If `True`, converts text to ASCII only.
87 """
88 # Store arguments for serialization
89 self.remove_empty_lines = remove_empty_lines
90 self.remove_extra_whitespaces = remove_extra_whitespaces
91 self.remove_repeated_substrings = remove_repeated_substrings
92 self.keep_id = keep_id
93 self.remove_substrings = remove_substrings
94 self.remove_regex = remove_regex
95 self.unicode_normalization = unicode_normalization
96 self.ascii_only = ascii_only
98 self.split_by = split_by
99 self.split_length = split_length
100 self.split_overlap = split_overlap
101 self.split_threshold = split_threshold
102 self.splitting_function = splitting_function
103 self.respect_sentence_boundary = respect_sentence_boundary
104 self.language = language
105 self.use_split_rules = use_split_rules
106 self.extend_abbreviations = extend_abbreviations
107 self.tokenizer_encoding = tokenizer_encoding
109 # Instantiate sub-components
110 splitter = DocumentSplitter(
111 split_by=self.split_by,
112 split_length=self.split_length,
113 split_overlap=self.split_overlap,
114 split_threshold=self.split_threshold,
115 splitting_function=self.splitting_function,
116 respect_sentence_boundary=self.respect_sentence_boundary,
117 language=self.language,
118 use_split_rules=self.use_split_rules,
119 extend_abbreviations=self.extend_abbreviations,
120 tokenizer_encoding=self.tokenizer_encoding,
121 )
123 cleaner = DocumentCleaner(
124 remove_empty_lines=self.remove_empty_lines,
125 remove_extra_whitespaces=self.remove_extra_whitespaces,
126 remove_repeated_substrings=self.remove_repeated_substrings,
127 keep_id=self.keep_id,
128 remove_substrings=self.remove_substrings,
129 remove_regex=self.remove_regex,
130 unicode_normalization=self.unicode_normalization,
131 ascii_only=self.ascii_only,
132 )
134 # Build the Pipeline
135 pp = Pipeline()
137 pp.add_component("splitter", splitter)
138 pp.add_component("cleaner", cleaner)
140 # Connect the splitter output to cleaner
141 pp.connect("splitter.documents", "cleaner.documents")
142 self.pipeline = pp
144 # Define how pipeline inputs/outputs map to sub-component inputs/outputs
145 self.input_mapping = {
146 # The pipeline input "documents" feeds into "splitter.documents"
147 "documents": ["splitter.documents"]
148 }
149 # The pipeline output "documents" comes from "cleaner.documents"
150 self.output_mapping = {"cleaner.documents": "documents"}
152 if TYPE_CHECKING:
153 # fake method, never executed, but static analyzers will not complain about missing method
154 def run(self, *, documents: list[Document]) -> dict[str, list[Document]]: # noqa: D102
155 ...
156 def warm_up(self) -> None: # noqa: D102
157 ...
159 def to_dict(self) -> dict[str, Any]:
160 """
161 Serialize SuperComponent to a dictionary.
163 :return:
164 Dictionary with serialized data.
165 """
166 splitting_function = None
167 if self.splitting_function is not None:
168 splitting_function = serialize_callable(self.splitting_function)
170 return default_to_dict(
171 self,
172 remove_empty_lines=self.remove_empty_lines,
173 remove_extra_whitespaces=self.remove_extra_whitespaces,
174 remove_repeated_substrings=self.remove_repeated_substrings,
175 keep_id=self.keep_id,
176 remove_substrings=self.remove_substrings,
177 remove_regex=self.remove_regex,
178 unicode_normalization=self.unicode_normalization,
179 ascii_only=self.ascii_only,
180 split_by=self.split_by,
181 split_length=self.split_length,
182 split_overlap=self.split_overlap,
183 split_threshold=self.split_threshold,
184 splitting_function=splitting_function,
185 respect_sentence_boundary=self.respect_sentence_boundary,
186 language=self.language,
187 use_split_rules=self.use_split_rules,
188 extend_abbreviations=self.extend_abbreviations,
189 tokenizer_encoding=self.tokenizer_encoding,
190 )
192 @classmethod
193 def from_dict(cls, data: dict[str, Any]) -> "DocumentPreprocessor":
194 """
195 Deserializes the SuperComponent from a dictionary.
197 :param data:
198 Dictionary to deserialize from.
199 :returns:
200 Deserialized SuperComponent.
201 """
202 splitting_function = data["init_parameters"].get("splitting_function", None)
203 if splitting_function:
204 data["init_parameters"]["splitting_function"] = deserialize_callable(splitting_function)
205 return default_from_dict(cls, data)