From ed131dbc163ff6dfe0596fa1bd718d86111bce6e Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Thu, 18 Jun 2026 14:10:23 +0200 Subject: [PATCH] chore: remove langdetect components (DocumentLanguageClassifier, TextLanguageRouter) (#11663) Co-authored-by: Claude Fable 5 --- MIGRATION.md | 5 +- haystack/components/classifiers/__init__.py | 16 --- .../document_language_classifier.py | 124 ------------------ haystack/components/routers/__init__.py | 2 - .../routers/text_language_router.py | 100 -------------- pydoc/classifiers_api.yml | 12 -- pydoc/routers_api.yml | 2 +- ...angdetect-components-b18af414c497a70c.yaml | 20 +++ .../test_document_language_classifier.py | 59 --------- .../routers/test_metadata_router.py | 5 +- .../routers/test_text_language_router.py | 58 -------- 11 files changed, 26 insertions(+), 377 deletions(-) delete mode 100644 haystack/components/classifiers/__init__.py delete mode 100644 haystack/components/classifiers/document_language_classifier.py delete mode 100644 haystack/components/routers/text_language_router.py delete mode 100644 pydoc/classifiers_api.yml create mode 100644 releasenotes/notes/remove-langdetect-components-b18af414c497a70c.yaml delete mode 100644 test/components/classifiers/test_document_language_classifier.py delete mode 100644 test/components/routers/test_text_language_router.py diff --git a/MIGRATION.md b/MIGRATION.md index 472ddc04be..872def6593 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -99,7 +99,10 @@ pip install | `from haystack.components.routers import TransformersTextRouter` | `transformers-haystack` | `from haystack_integrations.components.routers.transformers import TransformersTextRouter` | | `from haystack.components.routers import TransformersZeroShotTextRouter` | `transformers-haystack` | `from haystack_integrations.components.routers.transformers import TransformersZeroShotTextRouter` | | `from haystack.components.websearch import SerperDevWebSearch` | `serperdev-haystack` | `from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch` | -| `from haystack.components.websearch import SearchApiWebSearch` | `searchapi-haystack` | `from haystack_integrations.components.websearch.searchapi import SearchApiWebSearch` || `from haystack.components.extractors import NamedEntityExtractor` (Hugging Face backend) | `transformers-haystack` | `from haystack_integrations.components.extractors.transformers import TransformersNamedEntityExtractor` | +| `from haystack.components.websearch import SearchApiWebSearch` | `searchapi-haystack` | `from haystack_integrations.components.websearch.searchapi import SearchApiWebSearch` | +| `from haystack.components.classifiers import DocumentLanguageClassifier` | `langdetect-haystack` | `from haystack_integrations.components.classifiers.langdetect import DocumentLanguageClassifier` | +| `from haystack.components.routers import TextLanguageRouter` | `langdetect-haystack` | `from haystack_integrations.components.routers.langdetect import TextLanguageRouter` | +| `from haystack.components.extractors import NamedEntityExtractor` (Hugging Face backend) | `transformers-haystack` | `from haystack_integrations.components.extractors.transformers import TransformersNamedEntityExtractor` | | `from haystack.components.extractors import NamedEntityExtractor` (spaCy backend) | `spacy-haystack` | `from haystack_integrations.components.extractors.spacy import SpacyNamedEntityExtractor` | | `from haystack.components.embedders import SentenceTransformersTextEmbedder` | `sentence-transformers-haystack` | `from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder` | | `from haystack.components.embedders import SentenceTransformersDocumentEmbedder` | `sentence-transformers-haystack` | `from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder` | diff --git a/haystack/components/classifiers/__init__.py b/haystack/components/classifiers/__init__.py deleted file mode 100644 index e378ef471e..0000000000 --- a/haystack/components/classifiers/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import sys -from typing import TYPE_CHECKING - -from lazy_imports import LazyImporter - -_import_structure = {"document_language_classifier": ["DocumentLanguageClassifier"]} - -if TYPE_CHECKING: - from .document_language_classifier import DocumentLanguageClassifier as DocumentLanguageClassifier - -else: - sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure) diff --git a/haystack/components/classifiers/document_language_classifier.py b/haystack/components/classifiers/document_language_classifier.py deleted file mode 100644 index 64d4703d8a..0000000000 --- a/haystack/components/classifiers/document_language_classifier.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from dataclasses import replace - -from haystack import Document, component, logging -from haystack.lazy_imports import LazyImport - -logger = logging.getLogger(__name__) - -with LazyImport("Run 'pip install langdetect'") as langdetect_import: - import langdetect - - -@component -class DocumentLanguageClassifier: - """ - Classifies the language of each document and adds it to its metadata. - - Provide a list of languages during initialization. If the document's text doesn't match any of the - specified languages, the metadata value is set to "unmatched". - To route documents based on their language, use the MetadataRouter component after DocumentLanguageClassifier. - For routing plain text, use the TextLanguageRouter component instead. - - ### Usage example - - ```python - from haystack import Document, Pipeline - from haystack.document_stores.in_memory import InMemoryDocumentStore - from haystack.components.classifiers import DocumentLanguageClassifier - from haystack.components.routers import MetadataRouter - from haystack.components.writers import DocumentWriter - - docs = [Document(id="1", content="This is an English document"), - Document(id="2", content="Este es un documento en español")] - - document_store = InMemoryDocumentStore() - - p = Pipeline() - p.add_component(instance=DocumentLanguageClassifier(languages=["en"]), name="language_classifier") - p.add_component( - instance=MetadataRouter(rules={ - "en": { - "field": "meta.language", - "operator": "==", - "value": "en" - } - }), - name="router") - p.add_component(instance=DocumentWriter(document_store=document_store), name="writer") - p.connect("language_classifier.documents", "router.documents") - p.connect("router.en", "writer.documents") - - p.run({"language_classifier": {"documents": docs}}) - - written_docs = document_store.filter_documents() - assert len(written_docs) == 1 - assert written_docs[0] == Document(id="1", content="This is an English document", meta={"language": "en"}) - ``` - """ - - def __init__(self, languages: list[str] | None = None) -> None: - """ - Initializes the DocumentLanguageClassifier component. - - :param languages: A list of ISO language codes. - See the supported languages in [`langdetect` documentation](https://github.com/Mimino666/langdetect#languages). - If not specified, defaults to ["en"]. - """ - langdetect_import.check() - if not languages: - languages = ["en"] - self.languages = languages - - @component.output_types(documents=list[Document]) - def run(self, documents: list[Document]) -> dict[str, list[Document]]: - """ - Classifies the language of each document and adds it to its metadata. - - If the document's text doesn't match any of the languages specified at initialization, - sets the metadata value to "unmatched". - - :param documents: A list of documents for language classification. - - :returns: A dictionary with the following key: - - `documents`: A list of documents with an added `language` metadata field. - - :raises TypeError: if the input is not a list of Documents. - """ - if not isinstance(documents, list) or documents and not isinstance(documents[0], Document): - raise TypeError( - "DocumentLanguageClassifier expects a list of Document as input. " - "In case you want to classify and route a text, please use the TextLanguageRouter." - ) - - output: dict[str, list[Document]] = {language: [] for language in self.languages} - output["unmatched"] = [] - - new_documents = [] - for document in documents: - detected_language = self._detect_language(document) - new_meta = {**document.meta} - if detected_language in self.languages: - new_meta["language"] = detected_language - else: - new_meta["language"] = "unmatched" - new_documents.append(replace(document, meta=new_meta)) - - return {"documents": new_documents} - - def _detect_language(self, document: Document) -> str | None: - if document.content is None: - logger.warning( - "Langdetect cannot detect the language of Document with id: {document_id}", document_id=document.id - ) - return None - try: - return langdetect.detect(document.content) - except langdetect.LangDetectException: - logger.warning( - "Langdetect cannot detect the language of Document with id: {document_id}", document_id=document.id - ) - return None diff --git a/haystack/components/routers/__init__.py b/haystack/components/routers/__init__.py index 184c1519fc..cdb8d855b2 100644 --- a/haystack/components/routers/__init__.py +++ b/haystack/components/routers/__init__.py @@ -14,7 +14,6 @@ _import_structure = { "file_type_router": ["FileTypeRouter"], "llm_messages_router": ["LLMMessagesRouter"], "metadata_router": ["MetadataRouter"], - "text_language_router": ["TextLanguageRouter"], } if TYPE_CHECKING: @@ -24,6 +23,5 @@ if TYPE_CHECKING: from .file_type_router import FileTypeRouter as FileTypeRouter from .llm_messages_router import LLMMessagesRouter as LLMMessagesRouter from .metadata_router import MetadataRouter as MetadataRouter - from .text_language_router import TextLanguageRouter as TextLanguageRouter else: sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure) diff --git a/haystack/components/routers/text_language_router.py b/haystack/components/routers/text_language_router.py deleted file mode 100644 index d971e8c2d8..0000000000 --- a/haystack/components/routers/text_language_router.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from haystack import component, logging -from haystack.lazy_imports import LazyImport - -logger = logging.getLogger(__name__) - -with LazyImport("Run 'pip install langdetect'") as langdetect_import: - import langdetect - - -@component -class TextLanguageRouter: - """ - Routes text strings to different output connections based on their language. - - Provide a list of languages during initialization. If the document's text doesn't match any of the - specified languages, the metadata value is set to "unmatched". - For routing documents based on their language, use the DocumentLanguageClassifier component, - followed by the MetaDataRouter. - - ### Usage example - - ```python - from haystack import Pipeline, Document - from haystack.components.routers import TextLanguageRouter - from haystack.document_stores.in_memory import InMemoryDocumentStore - from haystack.components.retrievers.in_memory import InMemoryBM25Retriever - - document_store = InMemoryDocumentStore() - document_store.write_documents([Document(content="Elvis Presley was an American singer and actor.")]) - - p = Pipeline() - p.add_component(instance=TextLanguageRouter(languages=["en"]), name="text_language_router") - p.add_component(instance=InMemoryBM25Retriever(document_store=document_store), name="retriever") - p.connect("text_language_router.en", "retriever.query") - - result = p.run({"text_language_router": {"text": "Who was Elvis Presley?"}}) - assert result["retriever"]["documents"][0].content == "Elvis Presley was an American singer and actor." - - result = p.run({"text_language_router": {"text": "ένα ελληνικό κείμενο"}}) - assert result["text_language_router"]["unmatched"] == "ένα ελληνικό κείμενο" - ``` - """ - - def __init__(self, languages: list[str] | None = None) -> None: - """ - Initialize the TextLanguageRouter component. - - :param languages: A list of ISO language codes. - See the supported languages in [`langdetect` documentation](https://github.com/Mimino666/langdetect#languages). - If not specified, defaults to ["en"]. - """ - langdetect_import.check() - if not languages: - languages = ["en"] - self.languages = languages - component.set_output_types(self, unmatched=str, **dict.fromkeys(languages, str)) - - def run(self, text: str) -> dict[str, str]: - """ - Routes the text strings to different output connections based on their language. - - If the document's text doesn't match any of the specified languages, the metadata value is set to "unmatched". - - :param text: A text string to route. - - :returns: A dictionary in which the key is the language (or `"unmatched"`), - and the value is the text. - - :raises TypeError: If the input is not a string. - """ - if not isinstance(text, str): - msg = ( - "TextLanguageRouter expects a string as input. In case you want to classify a document, please use " - "the DocumentLanguageClassifier and MetaDataRouter." - ) - raise TypeError(msg) - - output: dict[str, str] = {} - - detected_language = self._detect_language(text) - if detected_language in self.languages: - output[detected_language] = text - else: - output["unmatched"] = text - - return output - - def _detect_language(self, text: str) -> str | None: - language = None - try: - language = langdetect.detect(text) - except langdetect.LangDetectException as exception: - logger.warning("Langdetect cannot detect the language of text. Error: {error}", error=exception) - # Only log the text in debug mode, as it might contain sensitive information - logger.debug("Langdetect cannot detect the language of text: {text}", text=text) - return language diff --git a/pydoc/classifiers_api.yml b/pydoc/classifiers_api.yml deleted file mode 100644 index 5225317780..0000000000 --- a/pydoc/classifiers_api.yml +++ /dev/null @@ -1,12 +0,0 @@ -loaders: - - search_path: [../haystack/components/classifiers] - modules: ["document_language_classifier"] -processors: - - type: filter - documented_only: true - skip_empty_modules: true -renderer: - title: Classifiers - id: classifiers-api - description: Classify documents based on the provided labels. - filename: classifiers_api.md diff --git a/pydoc/routers_api.yml b/pydoc/routers_api.yml index cd330f2a68..b1376612fe 100644 --- a/pydoc/routers_api.yml +++ b/pydoc/routers_api.yml @@ -1,7 +1,7 @@ loaders: - search_path: [../haystack/components/routers] modules: ["conditional_router", "document_length_router", "document_type_router", - "file_type_router", "llm_messages_router", "metadata_router", "text_language_router"] + "file_type_router", "llm_messages_router", "metadata_router"] processors: - type: filter documented_only: true diff --git a/releasenotes/notes/remove-langdetect-components-b18af414c497a70c.yaml b/releasenotes/notes/remove-langdetect-components-b18af414c497a70c.yaml new file mode 100644 index 0000000000..71aaa20a9e --- /dev/null +++ b/releasenotes/notes/remove-langdetect-components-b18af414c497a70c.yaml @@ -0,0 +1,20 @@ +--- +upgrade: + - | + ``DocumentLanguageClassifier`` and ``TextLanguageRouter`` have been moved out of Haystack into the + ``langdetect-haystack`` integration package. Install the new package with ``pip install langdetect-haystack`` + and update your imports. + + Before: + + .. code:: python + + from haystack.components.classifiers import DocumentLanguageClassifier + from haystack.components.routers import TextLanguageRouter + + After: + + .. code:: python + + from haystack_integrations.components.classifiers.langdetect import DocumentLanguageClassifier + from haystack_integrations.components.routers.langdetect import TextLanguageRouter diff --git a/test/components/classifiers/test_document_language_classifier.py b/test/components/classifiers/test_document_language_classifier.py deleted file mode 100644 index 6fe6e2cb89..0000000000 --- a/test/components/classifiers/test_document_language_classifier.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging - -import pytest - -from haystack import Document -from haystack.components.classifiers import DocumentLanguageClassifier - - -class TestDocumentLanguageClassifier: - def test_init(self): - component = DocumentLanguageClassifier() - assert component.languages == ["en"] - - def test_non_document_input(self): - with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."): - classifier = DocumentLanguageClassifier() - classifier.run(documents="This is an english sentence.") - - def test_single_document(self): - with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."): - classifier = DocumentLanguageClassifier() - classifier.run(documents=Document(content="This is an english sentence.")) - - def test_empty_list(self): - classifier = DocumentLanguageClassifier() - result = classifier.run(documents=[]) - assert result == {"documents": []} - - def test_detect_language(self): - classifier = DocumentLanguageClassifier() - detected_language = classifier._detect_language(Document(content="This is an english sentence.")) - assert detected_language == "en" - - def test_classify_as_en_and_unmatched(self): - classifier = DocumentLanguageClassifier() - english_document = Document(content="This is an english sentence.") - german_document = Document(content="Ein deutscher Satz ohne Verb.") - result = classifier.run(documents=[english_document, german_document]) - assert result["documents"][0].meta["language"] == "en" - assert result["documents"][1].meta["language"] == "unmatched" - assert "language" not in english_document.meta - assert "language" not in german_document.meta - - def test_warning_if_no_language_detected(self, caplog): - with caplog.at_level(logging.WARNING): - classifier = DocumentLanguageClassifier() - classifier.run(documents=[Document(content=".")]) - assert "Langdetect cannot detect the language of Document with id" in caplog.text - - def test_none_content_is_unmatched(self, caplog): - with caplog.at_level(logging.WARNING): - classifier = DocumentLanguageClassifier() - result = classifier.run(documents=[Document(content=None)]) - assert result["documents"][0].meta["language"] == "unmatched" - assert "Langdetect cannot detect the language of Document with id" in caplog.text diff --git a/test/components/routers/test_metadata_router.py b/test/components/routers/test_metadata_router.py index ce9af55dd4..a9cab6cbbb 100644 --- a/test/components/routers/test_metadata_router.py +++ b/test/components/routers/test_metadata_router.py @@ -5,7 +5,6 @@ import pytest from haystack import Pipeline -from haystack.components.classifiers import DocumentLanguageClassifier from haystack.components.routers.metadata_router import MetadataRouter from haystack.components.writers import DocumentWriter from haystack.dataclasses import ByteStream, Document @@ -198,13 +197,11 @@ class TestMetadataRouter: Document(content="Hello, welcome to the world of Haystack!", meta={"language": "en"}), Document(content="Hallo, willkommen in der Welt von Haystack!", meta={"language": "de"}), ] - p.add_component(instance=DocumentLanguageClassifier(), name="language_classifier") p.add_component( instance=MetadataRouter(rules={"en": {"field": "meta.language", "operator": "==", "value": "en"}}), name="router", ) p.add_component(instance=DocumentWriter(document_store=in_memory_doc_store), name="writer") - p.connect("language_classifier.documents", "router.documents") p.connect("router.en", "writer.documents") - p.run({"language_classifier": {"documents": docs}}) + p.run({"router": {"documents": docs}}) assert in_memory_doc_store.filter_documents() == [docs[0]] diff --git a/test/components/routers/test_text_language_router.py b/test/components/routers/test_text_language_router.py deleted file mode 100644 index 35eced7742..0000000000 --- a/test/components/routers/test_text_language_router.py +++ /dev/null @@ -1,58 +0,0 @@ -# SPDX-FileCopyrightText: 2022-present deepset GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging - -import pytest -from _pytest.logging import LogCaptureFixture - -from haystack import Document -from haystack.components.routers import TextLanguageRouter - - -class TestTextLanguageRouter: - def test_non_string_input(self): - with pytest.raises(TypeError, match="TextLanguageRouter expects a string as input."): - classifier = TextLanguageRouter() - classifier.run(text=Document(content="This is an english sentence.")) - - def test_list_of_string(self): - with pytest.raises(TypeError, match="TextLanguageRouter expects a string as input."): - classifier = TextLanguageRouter() - classifier.run(text=["This is an english sentence."]) - - def test_empty_string(self): - classifier = TextLanguageRouter() - result = classifier.run(text="") - assert result == {"unmatched": ""} - - def test_detect_language(self): - classifier = TextLanguageRouter() - detected_language = classifier._detect_language("This is an english sentence.") - assert detected_language == "en" - - def test_route_to_en(self): - classifier = TextLanguageRouter() - english_sentence = "This is an english sentence." - result = classifier.run(text=english_sentence) - assert result == {"en": english_sentence} - - def test_route_to_unmatched(self): - classifier = TextLanguageRouter() - german_sentence = "Ein deutscher Satz ohne Verb." - result = classifier.run(text=german_sentence) - assert result == {"unmatched": german_sentence} - - def test_warning_if_no_language_detected(self, caplog: LogCaptureFixture): - with caplog.at_level(logging.WARNING): - classifier = TextLanguageRouter() - classifier.run(text=".") - assert "Langdetect cannot detect the language of text. Error: No features in text." in caplog.text - - def test_warning_if_no_language_detected_if_debug(self, caplog: LogCaptureFixture): - with caplog.at_level(logging.DEBUG): - classifier = TextLanguageRouter() - classifier.run(text=".") - assert "Langdetect cannot detect the language of text. Error: No features in text." in caplog.text - assert "Langdetect cannot detect the language of text: ." in caplog.text