Coverage for haystack/components/preprocessors/recursive_splitter.py: 92%
266 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-12 09:42 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5import re
6from copy import deepcopy
7from typing import Any, Literal
9from haystack import Document, component, default_to_dict, logging
10from haystack.components.preprocessors._page_numbers import _leading_page_breaks
11from haystack.lazy_imports import LazyImport
13with LazyImport("Run 'pip install tiktoken'") as tiktoken_imports:
14 import tiktoken
16logger = logging.getLogger(__name__)
19@component
20class RecursiveDocumentSplitter:
21 """
22 Recursively chunk text into smaller chunks.
24 This component is used to split text into smaller chunks, it does so by recursively applying a list of separators
25 to the text.
27 The separators are applied in the order they are provided, typically this is a list of separators that are
28 applied in a specific order, being the last separator the most specific one.
30 Each separator is applied to the text, it then checks each of the resulting chunks, it keeps the chunks that
31 are within the split_length, for the ones that are larger than the split_length, it applies the next separator in the
32 list to the remaining text.
34 This is done until all chunks are smaller than the split_length parameter.
36 Example:
38 ```python
39 from haystack import Document
40 from haystack.components.preprocessors import RecursiveDocumentSplitter
42 chunker = RecursiveDocumentSplitter(split_length=15, split_overlap=0, separators=["\\n\\n", "\\n", ".", " "])
43 text = ('''Artificial intelligence (AI) - Introduction
45 AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
46 AI technology is widely used throughout industry, government, and science. Some high-profile applications include advanced web search engines; recommendation systems; interacting via human speech; autonomous vehicles; generative and creative tools; and superhuman play and analysis in strategy games.''')
47 doc = Document(content=text)
48 doc_chunks = chunker.run([doc])
49 print(doc_chunks["documents"])
50 # [
51 # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\\n\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': None, 'page_number': 1})
52 # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': None, 'page_number': 1})
53 # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': None, 'page_number': 1})
54 # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': None, 'page_number': 1})
55 # Document(id=..., content: 'vehicles; generative and creative tools; and superhuman play and analysis in strategy games.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 4, 'split_idx_start': 350, '_split_overlap': None, 'page_number': 1})
56 # ]
57 ```
58 """ # noqa: E501
60 def __init__(
61 self,
62 *,
63 split_length: int = 200,
64 split_overlap: int = 0,
65 split_unit: Literal["word", "char", "token"] = "word",
66 separators: list[str] | None = None,
67 sentence_splitter_params: dict[str, Any] | None = None,
68 ) -> None:
69 """
70 Initializes a RecursiveDocumentSplitter.
72 :param split_length: The maximum length of each chunk by default in words, but can be in characters or tokens.
73 See the `split_units` parameter.
74 :param split_overlap: The number of overlapping units (words, characters, or tokens, per
75 `split_unit`) between consecutive chunks.
76 :param split_unit: The unit of the split_length parameter. It can be either "word", "char", or "token".
77 If "token" is selected, the text will be split into tokens using the tiktoken tokenizer (o200k_base).
78 Special-token strings in document content are encoded as ordinary text.
79 :param separators: An optional list of separator strings to use for splitting the text. The string
80 separators will be treated as regular expressions unless the separator is "sentence", in that case the
81 text will be split into sentences using a custom sentence tokenizer based on NLTK.
82 See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter.
83 If no separators are provided, the default separators ["\\n\\n", "sentence", "\\n", " "] are used.
84 :param sentence_splitter_params: Optional parameters to pass to the sentence tokenizer.
85 See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter for more information.
86 The chunks keep the whitespace between sentences whether `keep_white_spaces` is True or False; it only
87 changes the rules the tokenizer uses to decide where a sentence ends.
89 :raises ValueError: If the overlap is greater than or equal to the chunk size or if the overlap is negative, or
90 if any separator is not a string.
91 """
92 self.split_length = split_length
93 self.split_overlap = split_overlap
94 self.split_units = split_unit
95 self.separators = separators if separators else ["\n\n", "sentence", "\n", " "] # default separators
96 self._check_params()
97 self.nltk_tokenizer = None
98 self.sentence_splitter_params = (
99 {"keep_white_spaces": True} if sentence_splitter_params is None else sentence_splitter_params
100 )
101 self.tiktoken_tokenizer: "tiktoken.Encoding" | None = None
102 self._is_warmed_up = False
104 def warm_up(self) -> None:
105 """
106 Warm up the sentence tokenizer and tiktoken tokenizer if needed.
107 """
108 if self._is_warmed_up:
109 return
110 if "sentence" in self.separators:
111 self.nltk_tokenizer = self._get_custom_sentence_tokenizer(self.sentence_splitter_params)
112 if self.split_units == "token":
113 tiktoken_imports.check()
114 self.tiktoken_tokenizer = tiktoken.get_encoding("o200k_base")
115 self._is_warmed_up = True
117 def to_dict(self) -> dict[str, Any]:
118 """
119 Serializes the component to a dictionary.
121 :returns:
122 Dictionary with serialized data.
123 """
124 return default_to_dict(
125 self,
126 split_length=self.split_length,
127 split_overlap=self.split_overlap,
128 split_unit=self.split_units,
129 separators=self.separators,
130 sentence_splitter_params=self.sentence_splitter_params,
131 )
133 def _check_params(self) -> None:
134 if self.split_length < 1:
135 raise ValueError("Split length must be at least 1 character.")
136 if self.split_overlap < 0:
137 raise ValueError("split_overlap must be greater than or equal to 0.")
138 if self.split_overlap >= self.split_length:
139 raise ValueError("Overlap cannot be greater than or equal to the chunk size.")
140 if not all(isinstance(separator, str) for separator in self.separators):
141 raise ValueError("All separators must be strings.")
143 @staticmethod
144 def _get_custom_sentence_tokenizer(sentence_splitter_params: dict[str, Any]) -> Any:
145 from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
147 return SentenceSplitter(**sentence_splitter_params)
149 def _split_chunk(self, current_chunk: str) -> tuple[str, str]:
150 """
151 Splits a chunk based on the split_length and split_units attribute.
153 :param current_chunk: The current chunk to be split.
154 :returns:
155 A tuple containing the current chunk and the remaining chunk.
156 """
157 if self.split_units == "char":
158 text = current_chunk
159 current_chunk = text[: self.split_length]
160 remaining_chars = text[self.split_length :]
161 return current_chunk, remaining_chars
163 # at this point we know that the tokenizer is already initialized
164 tokens = self.tiktoken_tokenizer.encode_ordinary(current_chunk) # type: ignore
165 current_tokens = tokens[: self.split_length]
166 remaining_tokens = tokens[self.split_length :]
167 return self.tiktoken_tokenizer.decode(current_tokens), self.tiktoken_tokenizer.decode(remaining_tokens) # type: ignore
169 def _apply_overlap(self, chunks: list[str]) -> list[str]:
170 """
171 Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
173 Works for character- and token-level splitting, words are handled by `_apply_word_overlap`. It trims the last
174 chunk if it exceeds the split_length and adds the trimmed content to the next chunk. If the last chunk is still
175 too long after trimming, it splits it and adds the first chunk to the list. This process continues until the
176 last chunk is within the split_length.
178 :param chunks: A list of text chunks.
179 :returns:
180 A list of text chunks with the overlap applied.
181 """
182 overlapped_chunks: list[str] = []
184 for idx, chunk in enumerate(chunks):
185 if idx == 0:
186 overlapped_chunks.append(chunk)
187 continue
189 # get the overlap between the current and previous chunk
190 overlap, prev_chunk = self._get_overlap(overlapped_chunks)
191 if overlap == prev_chunk:
192 logger.warning(
193 "Overlap is the same as the previous chunk. "
194 "Consider increasing the `split_length` parameter or decreasing the `split_overlap` parameter."
195 )
197 current_chunk = self._create_chunk_starting_with_overlap(chunk, overlap)
199 # if this new chunk exceeds 'split_length', trim it and move the remaining text to the next chunk
200 # if this is the last chunk, another new chunk will contain the trimmed text preceded by the overlap
201 # of the last chunk
202 if self._chunk_length(current_chunk) > self.split_length:
203 current_chunk, remaining_text = self._split_chunk(current_chunk)
204 if idx < len(chunks) - 1:
205 if self.split_units == "token":
206 # For token-based splitting, combine at token level
207 # at this point we know that the tokenizer is already initialized
208 remaining_tokens = self.tiktoken_tokenizer.encode_ordinary(remaining_text) # type: ignore
209 next_chunk_tokens = self.tiktoken_tokenizer.encode_ordinary(chunks[idx + 1]) # type: ignore
210 chunks[idx + 1] = self.tiktoken_tokenizer.decode(remaining_tokens + next_chunk_tokens) # type: ignore
211 else: # char
212 chunks[idx + 1] = remaining_text + chunks[idx + 1]
213 elif remaining_text:
214 # create a new chunk with the trimmed text preceded by the overlap of the last chunk
215 overlapped_chunks.append(current_chunk)
216 chunk = remaining_text
217 overlap, _ = self._get_overlap(overlapped_chunks)
218 current_chunk = self._create_chunk_starting_with_overlap(chunk, overlap)
220 overlapped_chunks.append(current_chunk)
222 # it can still be that the new last chunk exceeds the 'split_length'
223 # continue splitting until the last chunk is within 'split_length'
224 if idx == len(chunks) - 1 and self._chunk_length(current_chunk) > self.split_length:
225 last_chunk = overlapped_chunks.pop()
226 first_chunk, remaining_chunk = self._split_chunk(last_chunk)
227 overlapped_chunks.append(first_chunk)
229 while remaining_chunk:
230 # combine overlap with remaining chunk
231 overlap, _ = self._get_overlap(overlapped_chunks)
232 current = self._create_chunk_starting_with_overlap(remaining_chunk, overlap)
234 # if it fits within split_length we are done
235 if self._chunk_length(current) <= self.split_length:
236 overlapped_chunks.append(current)
237 break
239 # otherwise split it again
240 first_chunk, remaining_chunk = self._split_chunk(current)
241 overlapped_chunks.append(first_chunk)
243 return overlapped_chunks
245 def _create_chunk_starting_with_overlap(self, chunk: str, overlap: str) -> str:
246 if self.split_units == "token":
247 # For token-based splitting, combine at token level
248 # at this point we know that the tokenizer is already initialized
249 overlap_tokens = self.tiktoken_tokenizer.encode_ordinary(overlap) # type: ignore
250 chunk_tokens = self.tiktoken_tokenizer.encode_ordinary(chunk) # type: ignore
251 current_chunk = self.tiktoken_tokenizer.decode(overlap_tokens + chunk_tokens) # type: ignore
252 else: # char
253 current_chunk = overlap + chunk
254 return current_chunk
256 def _get_overlap(self, overlapped_chunks: list[str]) -> tuple[str, str]:
257 """Get the previous overlapped chunk instead of the original chunk."""
258 prev_chunk = overlapped_chunks[-1]
259 overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
261 if self.split_units == "token":
262 # For token-based splitting, handle overlap at token level
263 # at this point we know that the tokenizer is already initialized
264 tokens = self.tiktoken_tokenizer.encode_ordinary(prev_chunk) # type: ignore
265 overlap_tokens = tokens[overlap_start:]
266 overlap = self.tiktoken_tokenizer.decode(overlap_tokens) # type: ignore
267 else: # char
268 overlap = prev_chunk[overlap_start:]
270 return overlap, prev_chunk
272 def _apply_word_overlap(self, chunks: list[str]) -> list[tuple[int, int]]:
273 """
274 Applies an overlap between consecutive chunks when splitting by words.
276 Follows `_apply_overlap`: each chunk starts with the last `split_overlap` words of the chunk before it, and a
277 chunk that grows past `split_length` words is trimmed, its remaining words moving on to the next chunk. The
278 chunks are cut out of the text rather than rebuilt from their words, so they keep its whitespace and page
279 breaks. Words are counted as in `_chunk_length`.
281 :param chunks: A list of text chunks, as returned by `_chunk_text`.
282 :returns:
283 The `(start, end)` character offsets of each chunk with the overlap applied, in the concatenated `chunks`.
284 """
285 # The words of every chunk, as offsets into the concatenated chunks. A chunk of only spaces has no words and
286 # joins the chunk before it (or after it, at the start of the text).
287 words: list[tuple[int, int]] = []
288 chunk_word_ends: list[int] = []
289 chunk_starts: list[int] = []
290 offset = 0
291 for chunk in chunks:
292 chunk_words = [(offset + match.start(), offset + match.end()) for match in re.finditer(r"[^ ]+", chunk)]
293 if chunk_words:
294 words.extend(chunk_words)
295 chunk_word_ends.append(len(words))
296 chunk_starts.append(offset)
297 offset += len(chunk)
298 if not words:
299 return [(0, offset)]
300 # a chunk's text runs up to where the next one starts
301 chunk_ends = chunk_starts[1:] + [offset]
303 spans = [(0, chunk_ends[0])]
304 # the words the latest chunk spans, as indices into `words`
305 first_word, end_word = 0, chunk_word_ends[0]
306 for idx in range(1, len(chunk_word_ends)):
307 if end_word - first_word <= self.split_overlap:
308 logger.warning(
309 "Overlap is the same as the previous chunk. "
310 "Consider increasing the `split_length` parameter or decreasing the `split_overlap` parameter."
311 )
312 # only the last chunk emits more than one chunk; the others hand their remaining words to the next one
313 while True:
314 first_word = max(first_word, end_word - self.split_overlap)
315 end_word = min(first_word + self.split_length, chunk_word_ends[idx])
316 # a trimmed chunk ends after its last word, and the next chunk picks up from its overlap
317 end = chunk_ends[idx] if end_word == chunk_word_ends[idx] else words[end_word - 1][1]
318 spans.append((words[first_word][0], end))
319 if idx < len(chunk_word_ends) - 1 or end_word == chunk_word_ends[idx]:
320 break
322 return spans
324 def _chunk_length(self, text: str) -> int:
325 """
326 Get the length of the chunk in the specified units (words, characters, or tokens).
328 :param text: The text to measure.
329 :returns: The length of the text in the specified units.
330 """
331 if self.split_units == "word":
332 words = [word for word in text.split(" ") if word]
333 return len(words)
334 if self.split_units == "char":
335 return len(text)
336 # token
337 # at this point we know that the tokenizer is already initialized
338 return len(self.tiktoken_tokenizer.encode_ordinary(text)) # type: ignore
340 def _chunk_text(self, text: str) -> list[str]:
341 """
342 Recursive chunking algorithm that divides text into smaller chunks based on a list of separator characters.
344 It starts with a list of separator characters (e.g., ["\n\n", "sentence", "\n", " "]) and attempts to divide
345 the text using the first separator. If the resulting chunks are still larger than the specified chunk size,
346 it moves to the next separator in the list. This process continues recursively, progressively applying each
347 specific separator until the chunks meet the desired size criteria.
349 :param text: The text to be split into chunks.
350 :returns:
351 A list of text chunks.
352 """
353 if self._chunk_length(text) <= self.split_length:
354 return [text]
356 for curr_separator in self.separators:
357 # text after the last split that the splits leave out
358 trailing_text = ""
359 if curr_separator == "sentence":
360 # re. ignore: correct SentenceSplitter initialization is checked at the initialization of the component
361 sentence_with_spans = self.nltk_tokenizer.split_sentences(text) # type: ignore
362 # Each split runs from where its sentence starts to where the next one does, so the splits keep the
363 # whitespace between sentences even where the tokenizer leaves it out (keep_white_spaces=False).
364 # Leaving it out would glue sentences together, lose page breaks and shift every later chunk's offset.
365 split_bounds = [sentence["start"] for sentence in sentence_with_spans]
366 if split_bounds:
367 split_bounds.append(sentence_with_spans[-1]["end"])
368 trailing_text = text[split_bounds[-1] :]
369 splits = [text[start:end] for start, end in zip(split_bounds, split_bounds[1:], strict=False)]
370 else:
371 # add escape "\" to the separator and wrapped it in a group so that it's included in the splits as well
372 escaped_separator = re.escape(curr_separator)
373 escaped_separator = f"({escaped_separator})"
375 # split the text and merge every two consecutive splits, i.e.: the text and the separator after it
376 splits = re.split(escaped_separator, text)
377 splits = [
378 "".join([splits[i], splits[i + 1]]) if i < len(splits) - 1 else splits[i]
379 for i in range(0, len(splits), 2)
380 ]
382 # remove last split if it's empty
383 splits = splits[:-1] if splits[-1] == "" else splits
385 if len(splits) == 1: # go to next separator, if current separator not found in the text
386 continue
388 chunks = []
389 current_chunk: list[str] = []
390 current_length = 0
392 # check splits, if any is too long, recursively chunk it, otherwise add to current chunk
393 for split in splits:
394 split_text = split
396 # if adding this split exceeds chunk_size, process current_chunk
397 if current_length + self._chunk_length(split_text) > self.split_length:
398 # process current_chunk
399 if current_chunk: # keep the good splits
400 chunks.append("".join(current_chunk))
401 current_chunk = []
402 current_length = 0
404 # recursively handle splits that are too large
405 if self._chunk_length(split_text) > self.split_length:
406 if curr_separator == self.separators[-1]:
407 # tried last separator, can't split further, do a fixed-split based on word/character/token
408 fall_back_chunks = self._fall_back_to_fixed_chunking(split_text, self.split_units)
409 chunks.extend(fall_back_chunks)
410 else:
411 chunks.extend(self._chunk_text(split_text))
413 else:
414 current_chunk.append(split_text)
415 current_length += self._chunk_length(split_text)
416 else:
417 current_chunk.append(split_text)
418 current_length += self._chunk_length(split_text)
420 if current_chunk:
421 chunks.append("".join(current_chunk))
423 if chunks:
424 # like the whitespace left over by _fall_back_to_fixed_chunking, attach it to the last chunk rather
425 # than making it a chunk of its own
426 chunks[-1] += trailing_text
427 return chunks
429 # if no separator worked, fall back to word- or character-level chunking
430 return self._fall_back_to_fixed_chunking(text, self.split_units)
432 def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> list[str]:
433 """
434 Fall back to a fixed chunking approach if no separator works for the text.
436 Splits the text into smaller chunks based on the split_length and split_units attributes, either by words,
437 characters, or tokens.
439 :param text: The text to be split into chunks.
440 :param split_units: The unit of the split_length parameter. It can be either "word", "char", or "token".
441 :returns:
442 A list of text chunks.
443 """
444 chunks = []
446 if split_units == "word":
447 words = re.findall(r"\S+|\s+", text)
448 current_chunk = []
449 current_length = 0
451 for word in words:
452 # re.findall above also yields multi-character whitespace tokens (" ", "\t").
453 # Only count real words toward the length; any whitespace run is a separator.
454 if word.strip():
455 current_chunk.append(word)
456 current_length += 1
457 if current_length == self.split_length and current_chunk:
458 chunks.append("".join(current_chunk))
459 current_chunk = []
460 current_length = 0
461 else:
462 current_chunk.append(word)
464 if current_length > 0:
465 chunks.append("".join(current_chunk))
466 elif current_chunk:
467 # Only whitespace is left over (e.g. trailing whitespace): attach it to the
468 # previous chunk instead of emitting a whitespace-only chunk on its own.
469 if chunks:
470 chunks[-1] += "".join(current_chunk)
471 else:
472 chunks.append("".join(current_chunk))
473 elif split_units == "char":
474 for i in range(0, self._chunk_length(text), self.split_length):
475 chunks.append(text[i : i + self.split_length])
476 else: # token
477 # at this point we know that the tokenizer is already initialized
478 tokens = self.tiktoken_tokenizer.encode_ordinary(text) # type: ignore
479 for i in range(0, len(tokens), self.split_length):
480 chunk_tokens = tokens[i : i + self.split_length]
481 chunks.append(self.tiktoken_tokenizer.decode(chunk_tokens)) # type: ignore
482 return chunks
484 def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None:
485 prev_doc = new_docs[-1]
486 # curr_pos and split_idx_start are character offsets, so measure the
487 # overlap and range in characters too (not via _chunk_length, which returns a word/token count).
488 prev_doc_length = len(prev_doc.content) # type: ignore
489 overlap_length = prev_doc_length - (curr_pos - prev_doc.meta["split_idx_start"])
490 if overlap_length > 0:
491 prev_doc.meta["_split_overlap"].append({"doc_id": new_doc.id, "range": (0, overlap_length)})
492 new_doc.meta["_split_overlap"].append(
493 {"doc_id": prev_doc.id, "range": (prev_doc_length - overlap_length, prev_doc_length)}
494 )
496 def _run_one(self, doc: Document) -> list[Document]:
497 content = doc.content or "" # run() skips documents without content, so this is a non-empty string
498 chunks = self._chunk_text(content)
499 chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks # remove last empty chunk if it exists
501 # apply the overlap once, on the fully chunked list, so that chunks produced
502 # at inner recursion levels don't get the overlap applied a second time
503 if self.split_overlap > 0 and self.split_units == "word":
504 text = "".join(chunks)
505 spans = self._apply_word_overlap(chunks)
506 chunks = [text[start:end] for start, end in spans]
507 chunk_starts = [start for start, _ in spans]
508 else:
509 if self.split_overlap > 0:
510 chunks = self._apply_overlap(chunks)
511 # each chunk starts where the previous one ends, minus the overlap it repeats
512 chunk_starts = [0]
513 for chunk in chunks[:-1]:
514 overlap = self._get_overlap([chunk])[0] if self.split_overlap > 0 else ""
515 chunk_starts.append(chunk_starts[-1] + len(chunk) - len(overlap))
517 new_docs: list[Document] = []
518 # Chunk starts never decrease, so the page breaks before each chunk are counted on from the previous one
519 # rather than from the start of the document every time.
520 page_breaks_before = 0
522 for split_nr, (chunk, current_position) in enumerate(zip(chunks, chunk_starts, strict=True)):
523 if split_nr > 0:
524 page_breaks_before += content.count("\f", chunk_starts[split_nr - 1], current_position)
525 meta = deepcopy(doc.meta)
526 meta["source_id"] = doc.id
527 meta["parent_id"] = doc.id
528 meta["split_id"] = split_nr
529 meta["split_idx_start"] = current_position
530 meta["_split_overlap"] = [] if self.split_overlap > 0 else None
531 new_doc = Document(content=chunk, meta=meta)
533 # add overlap information to the previous and current doc
534 if split_nr > 0 and self.split_overlap > 0:
535 self._add_overlap_info(current_position, new_doc, new_docs)
537 # The page the chunk's first non-page-break character is on: breaks before the chunk, plus the
538 # ones it opens with.
539 new_doc.meta["page_number"] = 1 + page_breaks_before + _leading_page_breaks(chunk)
541 new_docs.append(new_doc)
543 return new_docs
545 @component.output_types(documents=list[Document])
546 def run(self, documents: list[Document]) -> dict[str, list[Document]]:
547 """
548 Split a list of documents into documents with smaller chunks of text.
550 :param documents: List of Documents to split.
551 :returns:
552 A dictionary containing a key "documents" with a List of Documents with smaller chunks of text corresponding
553 to the input documents. Each chunk carries a metadata field `page_number` with the page the chunk
554 starts on, counting form feed ("\f") characters in the original document.
555 """
556 if not self._is_warmed_up and ("sentence" in self.separators or self.split_units == "token"):
557 self.warm_up()
559 docs = []
560 for doc in documents:
561 if not doc.content or doc.content == "":
562 logger.warning("Document ID {doc_id} has an empty content. Skipping this document.", doc_id=doc.id)
563 continue
564 docs.extend(self._run_one(doc))
566 return {"documents": docs}