chore: remove langdetect components (DocumentLanguageClassifier, TextLanguageRouter) (#11663)

Co-authored-by: Claude Fable 5 <[email protected]>

MIGRATION.md

@@ -99,7 +99,10 @@ pip install <new-package>
 | `from haystack.components.routers import TransformersTextRouter` | `transformers-haystack` | `from haystack_integrations.components.routers.transformers import TransformersTextRouter` |
 | `from haystack.components.routers import TransformersZeroShotTextRouter` | `transformers-haystack` | `from haystack_integrations.components.routers.transformers import TransformersZeroShotTextRouter` |
 | `from haystack.components.websearch import SerperDevWebSearch` | `serperdev-haystack` | `from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch` |
-| `from haystack.components.websearch import SearchApiWebSearch` | `searchapi-haystack` | `from haystack_integrations.components.websearch.searchapi import SearchApiWebSearch` || `from haystack.components.extractors import NamedEntityExtractor` (Hugging Face backend) | `transformers-haystack` | `from haystack_integrations.components.extractors.transformers import TransformersNamedEntityExtractor` |
+| `from haystack.components.websearch import SearchApiWebSearch` | `searchapi-haystack` | `from haystack_integrations.components.websearch.searchapi import SearchApiWebSearch` |
+| `from haystack.components.classifiers import DocumentLanguageClassifier` | `langdetect-haystack` | `from haystack_integrations.components.classifiers.langdetect import DocumentLanguageClassifier` |
+| `from haystack.components.routers import TextLanguageRouter` | `langdetect-haystack` | `from haystack_integrations.components.routers.langdetect import TextLanguageRouter` |
+| `from haystack.components.extractors import NamedEntityExtractor` (Hugging Face backend) | `transformers-haystack` | `from haystack_integrations.components.extractors.transformers import TransformersNamedEntityExtractor` |
 | `from haystack.components.extractors import NamedEntityExtractor` (spaCy backend) | `spacy-haystack` | `from haystack_integrations.components.extractors.spacy import SpacyNamedEntityExtractor` |
 | `from haystack.components.embedders import SentenceTransformersTextEmbedder` | `sentence-transformers-haystack` | `from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersTextEmbedder` |
 | `from haystack.components.embedders import SentenceTransformersDocumentEmbedder` | `sentence-transformers-haystack` | `from haystack_integrations.components.embedders.sentence_transformers import SentenceTransformersDocumentEmbedder` |

haystack/components/classifiers/__init__.py

@@ -1,16 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <[email protected]>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import sys
-from typing import TYPE_CHECKING
-
-from lazy_imports import LazyImporter
-
-_import_structure = {"document_language_classifier": ["DocumentLanguageClassifier"]}
-
-if TYPE_CHECKING:
-    from .document_language_classifier import DocumentLanguageClassifier as DocumentLanguageClassifier
-
-else:
-    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/classifiers/document_language_classifier.py

@@ -1,124 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <[email protected]>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from dataclasses import replace
-
-from haystack import Document, component, logging
-from haystack.lazy_imports import LazyImport
-
-logger = logging.getLogger(__name__)
-
-with LazyImport("Run 'pip install langdetect'") as langdetect_import:
-    import langdetect
-
-
-@component
-class DocumentLanguageClassifier:
-    """
-    Classifies the language of each document and adds it to its metadata.
-
-    Provide a list of languages during initialization. If the document's text doesn't match any of the
-    specified languages, the metadata value is set to "unmatched".
-    To route documents based on their language, use the MetadataRouter component after DocumentLanguageClassifier.
-    For routing plain text, use the TextLanguageRouter component instead.
-
-    ### Usage example
-
-    ```python
-    from haystack import Document, Pipeline
-    from haystack.document_stores.in_memory import InMemoryDocumentStore
-    from haystack.components.classifiers import DocumentLanguageClassifier
-    from haystack.components.routers import MetadataRouter
-    from haystack.components.writers import DocumentWriter
-
-    docs = [Document(id="1", content="This is an English document"),
-            Document(id="2", content="Este es un documento en español")]
-
-    document_store = InMemoryDocumentStore()
-
-    p = Pipeline()
-    p.add_component(instance=DocumentLanguageClassifier(languages=["en"]), name="language_classifier")
-    p.add_component(
-    instance=MetadataRouter(rules={
-        "en": {
-            "field": "meta.language",
-            "operator": "==",
-            "value": "en"
-        }
-    }),
-    name="router")
-    p.add_component(instance=DocumentWriter(document_store=document_store), name="writer")
-    p.connect("language_classifier.documents", "router.documents")
-    p.connect("router.en", "writer.documents")
-
-    p.run({"language_classifier": {"documents": docs}})
-
-    written_docs = document_store.filter_documents()
-    assert len(written_docs) == 1
-    assert written_docs[0] == Document(id="1", content="This is an English document", meta={"language": "en"})
-    ```
-    """
-
-    def __init__(self, languages: list[str] | None = None) -> None:
-        """
-        Initializes the DocumentLanguageClassifier component.
-
-        :param languages: A list of ISO language codes.
-            See the supported languages in [`langdetect` documentation](https://github.com/Mimino666/langdetect#languages).
-            If not specified, defaults to ["en"].
-        """
-        langdetect_import.check()
-        if not languages:
-            languages = ["en"]
-        self.languages = languages
-
-    @component.output_types(documents=list[Document])
-    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
-        """
-        Classifies the language of each document and adds it to its metadata.
-
-        If the document's text doesn't match any of the languages specified at initialization,
-        sets the metadata value to "unmatched".
-
-        :param documents: A list of documents for language classification.
-
-        :returns: A dictionary with the following key:
-            - `documents`: A list of documents with an added `language` metadata field.
-
-        :raises TypeError: if the input is not a list of Documents.
-        """
-        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
-            raise TypeError(
-                "DocumentLanguageClassifier expects a list of Document as input. "
-                "In case you want to classify and route a text, please use the TextLanguageRouter."
-            )
-
-        output: dict[str, list[Document]] = {language: [] for language in self.languages}
-        output["unmatched"] = []
-
-        new_documents = []
-        for document in documents:
-            detected_language = self._detect_language(document)
-            new_meta = {**document.meta}
-            if detected_language in self.languages:
-                new_meta["language"] = detected_language
-            else:
-                new_meta["language"] = "unmatched"
-            new_documents.append(replace(document, meta=new_meta))
-
-        return {"documents": new_documents}
-
-    def _detect_language(self, document: Document) -> str | None:
-        if document.content is None:
-            logger.warning(
-                "Langdetect cannot detect the language of Document with id: {document_id}", document_id=document.id
-            )
-            return None
-        try:
-            return langdetect.detect(document.content)
-        except langdetect.LangDetectException:
-            logger.warning(
-                "Langdetect cannot detect the language of Document with id: {document_id}", document_id=document.id
-            )
-        return None

haystack/components/routers/__init__.py

@@ -14,7 +14,6 @@ _import_structure = {
     "file_type_router": ["FileTypeRouter"],
     "llm_messages_router": ["LLMMessagesRouter"],
     "metadata_router": ["MetadataRouter"],
-    "text_language_router": ["TextLanguageRouter"],
 }
 
 if TYPE_CHECKING:
@@ -24,6 +23,5 @@ if TYPE_CHECKING:
     from .file_type_router import FileTypeRouter as FileTypeRouter
     from .llm_messages_router import LLMMessagesRouter as LLMMessagesRouter
     from .metadata_router import MetadataRouter as MetadataRouter
-    from .text_language_router import TextLanguageRouter as TextLanguageRouter
 else:
     sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/routers/text_language_router.py

@@ -1,100 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <[email protected]>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from haystack import component, logging
-from haystack.lazy_imports import LazyImport
-
-logger = logging.getLogger(__name__)
-
-with LazyImport("Run 'pip install langdetect'") as langdetect_import:
-    import langdetect
-
-
-@component
-class TextLanguageRouter:
-    """
-    Routes text strings to different output connections based on their language.
-
-    Provide a list of languages during initialization. If the document's text doesn't match any of the
-    specified languages, the metadata value is set to "unmatched".
-    For routing documents based on their language, use the DocumentLanguageClassifier component,
-    followed by the MetaDataRouter.
-
-    ### Usage example
-
-    ```python
-    from haystack import Pipeline, Document
-    from haystack.components.routers import TextLanguageRouter
-    from haystack.document_stores.in_memory import InMemoryDocumentStore
-    from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
-
-    document_store = InMemoryDocumentStore()
-    document_store.write_documents([Document(content="Elvis Presley was an American singer and actor.")])
-
-    p = Pipeline()
-    p.add_component(instance=TextLanguageRouter(languages=["en"]), name="text_language_router")
-    p.add_component(instance=InMemoryBM25Retriever(document_store=document_store), name="retriever")
-    p.connect("text_language_router.en", "retriever.query")
-
-    result = p.run({"text_language_router": {"text": "Who was Elvis Presley?"}})
-    assert result["retriever"]["documents"][0].content == "Elvis Presley was an American singer and actor."
-
-    result = p.run({"text_language_router": {"text": "ένα ελληνικό κείμενο"}})
-    assert result["text_language_router"]["unmatched"] == "ένα ελληνικό κείμενο"
-    ```
-    """
-
-    def __init__(self, languages: list[str] | None = None) -> None:
-        """
-        Initialize the TextLanguageRouter component.
-
-        :param languages: A list of ISO language codes.
-            See the supported languages in [`langdetect` documentation](https://github.com/Mimino666/langdetect#languages).
-            If not specified, defaults to ["en"].
-        """
-        langdetect_import.check()
-        if not languages:
-            languages = ["en"]
-        self.languages = languages
-        component.set_output_types(self, unmatched=str, **dict.fromkeys(languages, str))
-
-    def run(self, text: str) -> dict[str, str]:
-        """
-        Routes the text strings to different output connections based on their language.
-
-        If the document's text doesn't match any of the specified languages, the metadata value is set to "unmatched".
-
-        :param text: A text string to route.
-
-        :returns: A dictionary in which the key is the language (or `"unmatched"`),
-            and the value is the text.
-
-        :raises TypeError: If the input is not a string.
-        """
-        if not isinstance(text, str):
-            msg = (
-                "TextLanguageRouter expects a string as input. In case you want to classify a document, please use "
-                "the DocumentLanguageClassifier and MetaDataRouter."
-            )
-            raise TypeError(msg)
-
-        output: dict[str, str] = {}
-
-        detected_language = self._detect_language(text)
-        if detected_language in self.languages:
-            output[detected_language] = text
-        else:
-            output["unmatched"] = text
-
-        return output
-
-    def _detect_language(self, text: str) -> str | None:
-        language = None
-        try:
-            language = langdetect.detect(text)
-        except langdetect.LangDetectException as exception:
-            logger.warning("Langdetect cannot detect the language of text. Error: {error}", error=exception)
-            # Only log the text in debug mode, as it might contain sensitive information
-            logger.debug("Langdetect cannot detect the language of text: {text}", text=text)
-        return language

pydoc/classifiers_api.yml

@@ -1,12 +0,0 @@
-loaders:
-  - search_path: [../haystack/components/classifiers]
-    modules: ["document_language_classifier"]
-processors:
-  - type: filter
-    documented_only: true
-    skip_empty_modules: true
-renderer:
-  title: Classifiers
-  id: classifiers-api
-  description: Classify documents based on the provided labels.
-  filename: classifiers_api.md

pydoc/routers_api.yml

@@ -1,7 +1,7 @@
 loaders:
   - search_path: [../haystack/components/routers]
     modules: ["conditional_router", "document_length_router", "document_type_router",
-      "file_type_router", "llm_messages_router", "metadata_router", "text_language_router"]
+      "file_type_router", "llm_messages_router", "metadata_router"]
 processors:
   - type: filter
     documented_only: true

releasenotes/notes/remove-langdetect-components-b18af414c497a70c.yaml

@@ -0,0 +1,20 @@
+---
+upgrade:
+  - |
+    ``DocumentLanguageClassifier`` and ``TextLanguageRouter`` have been moved out of Haystack into the
+    ``langdetect-haystack`` integration package. Install the new package with ``pip install langdetect-haystack``
+    and update your imports.
+
+    Before:
+
+    .. code:: python
+
+      from haystack.components.classifiers import DocumentLanguageClassifier
+      from haystack.components.routers import TextLanguageRouter
+
+    After:
+
+    .. code:: python
+
+      from haystack_integrations.components.classifiers.langdetect import DocumentLanguageClassifier
+      from haystack_integrations.components.routers.langdetect import TextLanguageRouter

test/components/classifiers/test_document_language_classifier.py

@@ -1,59 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <[email protected]>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-
-import pytest
-
-from haystack import Document
-from haystack.components.classifiers import DocumentLanguageClassifier
-
-
-class TestDocumentLanguageClassifier:
-    def test_init(self):
-        component = DocumentLanguageClassifier()
-        assert component.languages == ["en"]
-
-    def test_non_document_input(self):
-        with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."):
-            classifier = DocumentLanguageClassifier()
-            classifier.run(documents="This is an english sentence.")
-
-    def test_single_document(self):
-        with pytest.raises(TypeError, match="DocumentLanguageClassifier expects a list of Document as input."):
-            classifier = DocumentLanguageClassifier()
-            classifier.run(documents=Document(content="This is an english sentence."))
-
-    def test_empty_list(self):
-        classifier = DocumentLanguageClassifier()
-        result = classifier.run(documents=[])
-        assert result == {"documents": []}
-
-    def test_detect_language(self):
-        classifier = DocumentLanguageClassifier()
-        detected_language = classifier._detect_language(Document(content="This is an english sentence."))
-        assert detected_language == "en"
-
-    def test_classify_as_en_and_unmatched(self):
-        classifier = DocumentLanguageClassifier()
-        english_document = Document(content="This is an english sentence.")
-        german_document = Document(content="Ein deutscher Satz ohne Verb.")
-        result = classifier.run(documents=[english_document, german_document])
-        assert result["documents"][0].meta["language"] == "en"
-        assert result["documents"][1].meta["language"] == "unmatched"
-        assert "language" not in english_document.meta
-        assert "language" not in german_document.meta
-
-    def test_warning_if_no_language_detected(self, caplog):
-        with caplog.at_level(logging.WARNING):
-            classifier = DocumentLanguageClassifier()
-            classifier.run(documents=[Document(content=".")])
-            assert "Langdetect cannot detect the language of Document with id" in caplog.text
-
-    def test_none_content_is_unmatched(self, caplog):
-        with caplog.at_level(logging.WARNING):
-            classifier = DocumentLanguageClassifier()
-            result = classifier.run(documents=[Document(content=None)])
-            assert result["documents"][0].meta["language"] == "unmatched"
-            assert "Langdetect cannot detect the language of Document with id" in caplog.text

test/components/routers/test_metadata_router.py

@@ -5,7 +5,6 @@
 import pytest
 
 from haystack import Pipeline
-from haystack.components.classifiers import DocumentLanguageClassifier
 from haystack.components.routers.metadata_router import MetadataRouter
 from haystack.components.writers import DocumentWriter
 from haystack.dataclasses import ByteStream, Document
@@ -198,13 +197,11 @@ class TestMetadataRouter:
             Document(content="Hello, welcome to the world of Haystack!", meta={"language": "en"}),
             Document(content="Hallo, willkommen in der Welt von Haystack!", meta={"language": "de"}),
         ]
-        p.add_component(instance=DocumentLanguageClassifier(), name="language_classifier")
         p.add_component(
             instance=MetadataRouter(rules={"en": {"field": "meta.language", "operator": "==", "value": "en"}}),
             name="router",
         )
         p.add_component(instance=DocumentWriter(document_store=in_memory_doc_store), name="writer")
-        p.connect("language_classifier.documents", "router.documents")
         p.connect("router.en", "writer.documents")
-        p.run({"language_classifier": {"documents": docs}})
+        p.run({"router": {"documents": docs}})
         assert in_memory_doc_store.filter_documents() == [docs[0]]

test/components/routers/test_text_language_router.py

@@ -1,58 +0,0 @@
-# SPDX-FileCopyrightText: 2022-present deepset GmbH <[email protected]>
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import logging
-
-import pytest
-from _pytest.logging import LogCaptureFixture
-
-from haystack import Document
-from haystack.components.routers import TextLanguageRouter
-
-
-class TestTextLanguageRouter:
-    def test_non_string_input(self):
-        with pytest.raises(TypeError, match="TextLanguageRouter expects a string as input."):
-            classifier = TextLanguageRouter()
-            classifier.run(text=Document(content="This is an english sentence."))
-
-    def test_list_of_string(self):
-        with pytest.raises(TypeError, match="TextLanguageRouter expects a string as input."):
-            classifier = TextLanguageRouter()
-            classifier.run(text=["This is an english sentence."])
-
-    def test_empty_string(self):
-        classifier = TextLanguageRouter()
-        result = classifier.run(text="")
-        assert result == {"unmatched": ""}
-
-    def test_detect_language(self):
-        classifier = TextLanguageRouter()
-        detected_language = classifier._detect_language("This is an english sentence.")
-        assert detected_language == "en"
-
-    def test_route_to_en(self):
-        classifier = TextLanguageRouter()
-        english_sentence = "This is an english sentence."
-        result = classifier.run(text=english_sentence)
-        assert result == {"en": english_sentence}
-
-    def test_route_to_unmatched(self):
-        classifier = TextLanguageRouter()
-        german_sentence = "Ein deutscher Satz ohne Verb."
-        result = classifier.run(text=german_sentence)
-        assert result == {"unmatched": german_sentence}
-
-    def test_warning_if_no_language_detected(self, caplog: LogCaptureFixture):
-        with caplog.at_level(logging.WARNING):
-            classifier = TextLanguageRouter()
-            classifier.run(text=".")
-            assert "Langdetect cannot detect the language of text. Error: No features in text." in caplog.text
-
-    def test_warning_if_no_language_detected_if_debug(self, caplog: LogCaptureFixture):
-        with caplog.at_level(logging.DEBUG):
-            classifier = TextLanguageRouter()
-            classifier.run(text=".")
-            assert "Langdetect cannot detect the language of text. Error: No features in text." in caplog.text
-            assert "Langdetect cannot detect the language of text: ." in caplog.text