chore: adopt PEP 585 type hints (#9678)

* chore(lint): enforce and apply PEP 585 type hinting

* Run fmt fixes

* Fix all typing imports using some regex

* Fix all typing written in string in tests

* undo changes in the e2e tests

* make e2e test use list instead of List

* type fixes

* remove type:ignore

* pylint

* Remove typing from Usage example comments

* Remove typing from most of comments

* try to fix e2e tests on comm PRs

* fix

* Add tests typing.List in to adjust test compatiplity
- test/components/agents/test_state_class.py
- test/components/converters/test_output_adapter.py
- test/components/joiners/test_list_joiner.py

* simplify pyproject

* improve relnote

---------

Co-authored-by: anakin87 <[email protected]>
This commit is contained in:
Abdelrahman Kaseb
2025-08-07 10:23:14 +02:00
committed by GitHub
co-authored by anakin87
parent 4b9fb20bab
commit 5f3c37d287
227 changed files with 2092 additions and 2005 deletions
+2 -2
View File
@@ -3,12 +3,12 @@ import os
import sys
import traceback
from pathlib import Path
from typing import List, Optional
from typing import Optional
from haystack import logging # pylint: disable=unused-import # this is needed to avoid circular imports
def validate_module_imports(root_dir: str, exclude_subdirs: Optional[List[str]] = None) -> tuple[list, list]:
def validate_module_imports(root_dir: str, exclude_subdirs: Optional[list[str]] = None) -> tuple[list, list]:
"""
Recursively search for all Python modules and attempt to import them.
+1 -2
View File
@@ -3,7 +3,6 @@ import base64
import os
import re
from pathlib import Path
from typing import List
import requests
import yaml
@@ -30,7 +29,7 @@ def create_headers(version: str):
return {"authorization": f"Basic {readme_token()}", "x-readme-version": version}
def get_docs_in_category(category_slug: str, version: str) -> List[str]:
def get_docs_in_category(category_slug: str, version: str) -> list[str]:
"""
Returns the slugs of all documents in a category for the specific version.
"""
+4 -1
View File
@@ -15,7 +15,10 @@ from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
def test_dense_doc_search_pipeline(tmp_path, samples_path):
def test_dense_doc_search_pipeline(tmp_path, samples_path, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
# Create the indexing pipeline
indexing_pipeline = Pipeline()
indexing_pipeline.add_component(
+1 -2
View File
@@ -3,7 +3,6 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import List
import pytest
@@ -30,7 +29,7 @@ from haystack.dataclasses import ChatMessage
EMBEDDINGS_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
def indexing_pipeline(documents: List[Document]):
def indexing_pipeline(documents: list[Document]):
"""Indexing the documents"""
document_store = InMemoryDocumentStore()
doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP)
+4 -1
View File
@@ -8,7 +8,10 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.document_stores.in_memory import InMemoryDocumentStore
def test_extractive_qa_pipeline(tmp_path):
def test_extractive_qa_pipeline(tmp_path, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
# Create the pipeline
qa_pipeline = Pipeline()
qa_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=InMemoryDocumentStore()), name="retriever")
@@ -11,7 +11,10 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMe
from haystack.document_stores.in_memory import InMemoryDocumentStore
def test_hybrid_doc_search_pipeline(tmp_path):
def test_hybrid_doc_search_pipeline(tmp_path, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
# Create the pipeline
document_store = InMemoryDocumentStore()
hybrid_pipeline = Pipeline()
+12 -3
View File
@@ -47,7 +47,10 @@ def spacy_annotations():
]
def test_ner_extractor_init():
def test_ner_extractor_init(monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
extractor = NamedEntityExtractor(backend=NamedEntityExtractorBackend.HUGGING_FACE, model="dslim/bert-base-NER")
with pytest.raises(RuntimeError, match=r"not warmed up"):
@@ -59,7 +62,10 @@ def test_ner_extractor_init():
@pytest.mark.parametrize("batch_size", [1, 3])
def test_ner_extractor_hf_backend(raw_texts, hf_annotations, batch_size):
def test_ner_extractor_hf_backend(raw_texts, hf_annotations, batch_size, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
extractor = NamedEntityExtractor(backend=NamedEntityExtractorBackend.HUGGING_FACE, model="dslim/bert-base-NER")
extractor.warm_up()
@@ -87,7 +93,10 @@ def test_ner_extractor_spacy_backend(raw_texts, spacy_annotations, batch_size):
@pytest.mark.parametrize("batch_size", [1, 3])
def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size):
def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
pipeline = Pipeline()
pipeline.add_component(
name="ner_extractor",
@@ -2,6 +2,10 @@
#
# SPDX-License-Identifier: Apache-2.0
import os
import pytest
from haystack import Pipeline
from haystack.components.converters.pypdf import PyPDFToDocument
from haystack.components.joiners import DocumentJoiner
@@ -13,6 +17,10 @@ from haystack.components.generators.chat.openai import OpenAIChatGenerator
from haystack.components.routers.document_length_router import DocumentLengthRouter
@pytest.mark.skipif(
not os.environ.get("OPENAI_API_KEY", None),
reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
)
def test_pdf_content_extraction_pipeline():
"""
Test a pipeline that processes PDFs with the following steps:
+4 -1
View File
@@ -12,7 +12,10 @@ from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
def test_preprocessing_pipeline(tmp_path):
def test_preprocessing_pipeline(tmp_path, monkeypatch):
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
# Create the pipeline and its components
document_store = InMemoryDocumentStore()
preprocessing_pipeline = Pipeline()
+15 -15
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import inspect
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import logging, tracing
from haystack.components.generators.chat.types import ChatGenerator
@@ -71,14 +71,14 @@ class Agent:
self,
*,
chat_generator: ChatGenerator,
tools: Optional[Union[List[Tool], Toolset]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
system_prompt: Optional[str] = None,
exit_conditions: Optional[List[str]] = None,
state_schema: Optional[Dict[str, Any]] = None,
exit_conditions: Optional[list[str]] = None,
state_schema: Optional[dict[str, Any]] = None,
max_agent_steps: int = 100,
streaming_callback: Optional[StreamingCallbackT] = None,
raise_on_tool_invocation_failure: bool = False,
tool_invoker_kwargs: Optional[Dict[str, Any]] = None,
tool_invoker_kwargs: Optional[dict[str, Any]] = None,
) -> None:
"""
Initialize the agent component.
@@ -126,7 +126,7 @@ class Agent:
# Initialize state schema
resolved_state_schema = _deepcopy_with_exceptions(self._state_schema)
if resolved_state_schema.get("messages") is None:
resolved_state_schema["messages"] = {"type": List[ChatMessage], "handler": merge_lists}
resolved_state_schema["messages"] = {"type": list[ChatMessage], "handler": merge_lists}
self.state_schema = resolved_state_schema
self.chat_generator = chat_generator
@@ -172,7 +172,7 @@ class Agent:
self.chat_generator.warm_up()
self._is_warmed_up = True
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize the component to a dictionary.
@@ -198,7 +198,7 @@ class Agent:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Agent":
def from_dict(cls, data: dict[str, Any]) -> "Agent":
"""
Deserialize the agent from a dictionary.
@@ -219,9 +219,9 @@ class Agent:
return default_from_dict(cls, data)
def _prepare_generator_inputs(self, streaming_callback: Optional[StreamingCallbackT] = None) -> Dict[str, Any]:
def _prepare_generator_inputs(self, streaming_callback: Optional[StreamingCallbackT] = None) -> dict[str, Any]:
"""Prepare inputs for the chat generator."""
generator_inputs: Dict[str, Any] = {"tools": self.tools}
generator_inputs: dict[str, Any] = {"tools": self.tools}
if streaming_callback is not None:
generator_inputs["streaming_callback"] = streaming_callback
return generator_inputs
@@ -240,13 +240,13 @@ class Agent:
def run( # noqa: PLR0915
self,
messages: List[ChatMessage],
messages: list[ChatMessage],
streaming_callback: Optional[StreamingCallbackT] = None,
*,
break_point: Optional[AgentBreakpoint] = None,
snapshot: Optional[AgentSnapshot] = None,
**kwargs: Any,
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""
Process messages and execute tools until an exit condition is met.
@@ -431,13 +431,13 @@ class Agent:
async def run_async( # noqa: PLR0915
self,
messages: List[ChatMessage],
messages: list[ChatMessage],
streaming_callback: Optional[StreamingCallbackT] = None,
*,
break_point: Optional[AgentBreakpoint] = None,
snapshot: Optional[AgentSnapshot] = None,
**kwargs: Any,
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""
Asynchronously process messages and execute tools until the exit condition is met.
@@ -626,7 +626,7 @@ class Agent:
result.update({"last_message": all_messages[-1]})
return result
def _check_exit_conditions(self, llm_messages: List[ChatMessage], tool_messages: List[ChatMessage]) -> bool:
def _check_exit_conditions(self, llm_messages: list[ChatMessage], tool_messages: list[ChatMessage]) -> bool:
"""
Check if any of the LLM messages' tool calls match an exit condition and if there are no errors.
+14 -9
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from copy import deepcopy
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Optional, get_args
from haystack.dataclasses import ChatMessage
from haystack.utils import _deserialize_value_with_schema, _serialize_value_with_schema
@@ -13,7 +13,7 @@ from haystack.utils.type_serialization import deserialize_type, serialize_type
from .state_utils import _is_list_type, _is_valid_type, merge_lists, replace_values
def _schema_to_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
def _schema_to_dict(schema: dict[str, Any]) -> dict[str, Any]:
"""
Convert a schema dictionary to a serializable format.
@@ -32,7 +32,7 @@ def _schema_to_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
return serialized_schema
def _schema_from_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
def _schema_from_dict(schema: dict[str, Any]) -> dict[str, Any]:
"""
Convert a serialized schema dictionary back to its original format.
@@ -52,7 +52,7 @@ def _schema_from_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
return deserialized_schema
def _validate_schema(schema: Dict[str, Any]) -> None:
def _validate_schema(schema: dict[str, Any]) -> None:
"""
Validate that a schema dictionary meets all required constraints.
@@ -69,8 +69,13 @@ def _validate_schema(schema: Dict[str, Any]) -> None:
raise ValueError(f"StateSchema: 'type' for key '{param}' must be a Python type, got {definition['type']}")
if definition.get("handler") is not None and not callable(definition["handler"]):
raise ValueError(f"StateSchema: 'handler' for key '{param}' must be callable or None")
if param == "messages" and definition["type"] != List[ChatMessage]:
raise ValueError(f"StateSchema: 'messages' must be of type List[ChatMessage], got {definition['type']}")
if param == "messages": # definition["type"] != list[ChatMessage] but split to cover also List[ChatMessage]
if not _is_list_type(definition["type"]):
raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
# Check if the list contains ChatMessage elements
args = get_args(definition["type"])
if not args or args[0] != ChatMessage:
raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
class State:
@@ -84,7 +89,7 @@ class State:
}
"""
def __init__(self, schema: Dict[str, Any], data: Optional[Dict[str, Any]] = None):
def __init__(self, schema: dict[str, Any], data: Optional[dict[str, Any]] = None):
"""
Initialize a State object with a schema and optional data.
@@ -98,7 +103,7 @@ class State:
_validate_schema(schema)
self.schema = deepcopy(schema)
if self.schema.get("messages") is None:
self.schema["messages"] = {"type": List[ChatMessage], "handler": merge_lists}
self.schema["messages"] = {"type": list[ChatMessage], "handler": merge_lists}
self._data = data or {}
# Set default handlers if not provided in schema
@@ -170,7 +175,7 @@ class State:
return serialized
@classmethod
def from_dict(cls, data: Dict[str, Any]):
def from_dict(cls, data: dict[str, Any]):
"""
Convert a dictionary back to a State object.
"""
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import inspect
from typing import Any, List, TypeVar, Union, get_origin
from typing import Any, TypeVar, Union, get_origin
T = TypeVar("T")
@@ -14,7 +14,7 @@ def _is_valid_type(obj: Any) -> bool:
Valid types include:
- Normal classes (str, dict, CustomClass)
- Generic types (List[str], Dict[str, int])
- Generic types (list[str], dict[str, int])
- Union types (Union[str, int], Optional[str])
:param obj: The object to check
@@ -23,7 +23,7 @@ def _is_valid_type(obj: Any) -> bool:
Example usage:
>>> _is_valid_type(str)
True
>>> _is_valid_type(List[int])
>>> _is_valid_type(list[int])
True
>>> _is_valid_type(Union[str, int])
True
@@ -48,7 +48,7 @@ def _is_list_type(type_hint: Any) -> bool:
return type_hint == list or (hasattr(type_hint, "__origin__") and get_origin(type_hint) == list)
def merge_lists(current: Union[List[T], T, None], new: Union[List[T], T]) -> List[T]:
def merge_lists(current: Union[list[T], T, None], new: Union[list[T], T]) -> list[T]:
"""
Merges two values into a single list.
+8 -8
View File
@@ -4,7 +4,7 @@
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Union, get_args
from typing import Any, Literal, Optional, Union, get_args
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.dataclasses import ByteStream
@@ -54,7 +54,7 @@ class LocalWhisperTranscriber:
self,
model: WhisperLocalModel = "large",
device: Optional[ComponentDevice] = None,
whisper_params: Optional[Dict[str, Any]] = None,
whisper_params: Optional[dict[str, Any]] = None,
):
"""
Creates an instance of the LocalWhisperTranscriber component.
@@ -84,7 +84,7 @@ class LocalWhisperTranscriber:
if not self._model:
self._model = whisper.load_model(self.model, device=self.device.to_torch())
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -94,7 +94,7 @@ class LocalWhisperTranscriber:
return default_to_dict(self, model=self.model, device=self.device.to_dict(), whisper_params=self.whisper_params)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "LocalWhisperTranscriber":
def from_dict(cls, data: dict[str, Any]) -> "LocalWhisperTranscriber":
"""
Deserializes the component from a dictionary.
@@ -108,8 +108,8 @@ class LocalWhisperTranscriber:
init_params["device"] = ComponentDevice.from_dict(init_params["device"])
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
def run(self, sources: List[Union[str, Path, ByteStream]], whisper_params: Optional[Dict[str, Any]] = None):
@component.output_types(documents=list[Document])
def run(self, sources: list[Union[str, Path, ByteStream]], whisper_params: Optional[dict[str, Any]] = None):
"""
Transcribes a list of audio files into a list of documents.
@@ -137,7 +137,7 @@ class LocalWhisperTranscriber:
documents = self.transcribe(sources, **whisper_params)
return {"documents": documents}
def transcribe(self, sources: List[Union[str, Path, ByteStream]], **kwargs) -> List[Document]:
def transcribe(self, sources: list[Union[str, Path, ByteStream]], **kwargs) -> list[Document]:
"""
Transcribes the audio files into a list of Documents, one for each input file.
@@ -158,7 +158,7 @@ class LocalWhisperTranscriber:
documents.append(doc)
return documents
def _raw_transcribe(self, sources: List[Union[str, Path, ByteStream]], **kwargs) -> Dict[Path, Any]:
def _raw_transcribe(self, sources: list[Union[str, Path, ByteStream]], **kwargs) -> dict[Path, Any]:
"""
Transcribes the given audio files. Returns the output of the model, a dictionary, for each input file.
+6 -6
View File
@@ -4,7 +4,7 @@
import io
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from openai import OpenAI
@@ -42,7 +42,7 @@ class RemoteWhisperTranscriber:
model: str = "whisper-1",
api_base_url: Optional[str] = None,
organization: Optional[str] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
**kwargs,
):
"""
@@ -105,7 +105,7 @@ class RemoteWhisperTranscriber:
http_client=init_http_client(self.http_client_kwargs, async_client=False),
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -123,7 +123,7 @@ class RemoteWhisperTranscriber:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "RemoteWhisperTranscriber":
def from_dict(cls, data: dict[str, Any]) -> "RemoteWhisperTranscriber":
"""
Deserializes the component from a dictionary.
@@ -135,8 +135,8 @@ class RemoteWhisperTranscriber:
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
def run(self, sources: List[Union[str, Path, ByteStream]]):
@component.output_types(documents=list[Document])
def run(self, sources: list[Union[str, Path, ByteStream]]):
"""
Transcribes the list of audio files into a list of documents.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import re
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, GeneratedAnswer, component, logging
from haystack.dataclasses.chat_message import ChatMessage
@@ -64,13 +64,13 @@ class AnswerBuilder:
self.reference_pattern = reference_pattern
self.last_message_only = last_message_only
@component.output_types(answers=List[GeneratedAnswer])
@component.output_types(answers=list[GeneratedAnswer])
def run( # pylint: disable=too-many-positional-arguments
self,
query: str,
replies: Union[List[str], List[ChatMessage]],
meta: Optional[List[Dict[str, Any]]] = None,
documents: Optional[List[Document]] = None,
replies: Union[list[str], list[ChatMessage]],
meta: Optional[list[dict[str, Any]]] = None,
documents: Optional[list[Document]] = None,
pattern: Optional[str] = None,
reference_pattern: Optional[str] = None,
):
@@ -184,7 +184,7 @@ class AnswerBuilder:
return ""
@staticmethod
def _extract_reference_idxs(reply: str, reference_pattern: str) -> List[int]:
def _extract_reference_idxs(reply: str, reference_pattern: str) -> list[int]:
document_idxs = re.findall(reference_pattern, reply)
return [int(idx) - 1 for idx in document_idxs]
@@ -4,7 +4,7 @@
import json
from copy import deepcopy
from typing import Any, Dict, List, Literal, Optional, Set, Union
from typing import Any, Literal, Optional, Union
from jinja2 import meta
from jinja2.sandbox import SandboxedEnvironment
@@ -140,9 +140,9 @@ class ChatPromptBuilder:
def __init__(
self,
template: Optional[Union[List[ChatMessage], str]] = None,
required_variables: Optional[Union[List[str], Literal["*"]]] = None,
variables: Optional[List[str]] = None,
template: Optional[Union[list[ChatMessage], str]] = None,
required_variables: Optional[Union[list[str], Literal["*"]]] = None,
variables: Optional[list[str]] = None,
):
"""
Constructs a ChatPromptBuilder component.
@@ -205,11 +205,11 @@ class ChatPromptBuilder:
else:
component.set_input_type(self, var, Any, "")
@component.output_types(prompt=List[ChatMessage])
@component.output_types(prompt=list[ChatMessage])
def run(
self,
template: Optional[Union[List[ChatMessage], str]] = None,
template_variables: Optional[Dict[str, Any]] = None,
template: Optional[Union[list[ChatMessage], str]] = None,
template_variables: Optional[dict[str, Any]] = None,
**kwargs,
):
"""
@@ -277,8 +277,8 @@ class ChatPromptBuilder:
return {"prompt": processed_messages}
def _render_chat_messages_from_str_template(
self, template: str, template_variables: Dict[str, Any]
) -> List[ChatMessage]:
self, template: str, template_variables: dict[str, Any]
) -> list[ChatMessage]:
"""
Renders a chat message from a string template.
@@ -296,7 +296,7 @@ class ChatPromptBuilder:
return messages
def _validate_variables(self, provided_variables: Set[str]):
def _validate_variables(self, provided_variables: set[str]):
"""
Checks if all the required template variables are provided.
@@ -317,14 +317,14 @@ class ChatPromptBuilder:
f"Required variables: {required_variables}. Provided variables: {provided_variables}."
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Returns a dictionary representation of the component.
:returns:
Serialized dictionary representation of the component.
"""
template: Optional[Union[List[Dict[str, Any]], str]] = None
template: Optional[Union[list[dict[str, Any]], str]] = None
if isinstance(self.template, list):
template = [m.to_dict() for m in self.template]
elif isinstance(self.template, str):
@@ -335,7 +335,7 @@ class ChatPromptBuilder:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ChatPromptBuilder":
def from_dict(cls, data: dict[str, Any]) -> "ChatPromptBuilder":
"""
Deserialize this component from a dictionary.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Optional, Set, Union
from typing import Any, Literal, Optional, Union
from jinja2 import meta
from jinja2.sandbox import SandboxedEnvironment
@@ -141,8 +141,8 @@ class PromptBuilder:
def __init__(
self,
template: str,
required_variables: Optional[Union[List[str], Literal["*"]]] = None,
variables: Optional[List[str]] = None,
required_variables: Optional[Union[list[str], Literal["*"]]] = None,
variables: Optional[list[str]] = None,
):
"""
Constructs a PromptBuilder component.
@@ -198,7 +198,7 @@ class PromptBuilder:
else:
component.set_input_type(self, var, Any, "")
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Returns a dictionary representation of the component.
@@ -210,7 +210,7 @@ class PromptBuilder:
)
@component.output_types(prompt=str)
def run(self, template: Optional[str] = None, template_variables: Optional[Dict[str, Any]] = None, **kwargs):
def run(self, template: Optional[str] = None, template_variables: Optional[dict[str, Any]] = None, **kwargs):
"""
Renders the prompt template with the provided variables.
@@ -244,7 +244,7 @@ class PromptBuilder:
result = compiled_template.render(template_variables_combined)
return {"prompt": result}
def _validate_variables(self, provided_variables: Set[str]):
def _validate_variables(self, provided_variables: set[str]):
"""
Checks if all the required template variables are provided.
+5 -5
View File
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List
from typing import Any
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.document_stores.types import DocumentStore
@@ -51,7 +51,7 @@ class CacheChecker:
self.document_store = document_store
self.cache_field = cache_field
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -61,7 +61,7 @@ class CacheChecker:
return default_to_dict(self, document_store=self.document_store.to_dict(), cache_field=self.cache_field)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "CacheChecker":
def from_dict(cls, data: dict[str, Any]) -> "CacheChecker":
"""
Deserializes the component from a dictionary.
@@ -75,8 +75,8 @@ class CacheChecker:
return default_from_dict(cls, data)
@component.output_types(hits=List[Document], misses=List)
def run(self, items: List[Any]):
@component.output_types(hits=list[Document], misses=list)
def run(self, items: list[Any]):
"""
Checks if any document associated with the specified cache field is already present in the store.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Dict, List, Optional
from typing import Optional
from haystack import Document, component, logging
from haystack.lazy_imports import LazyImport
@@ -52,7 +52,7 @@ class DocumentLanguageClassifier:
```
"""
def __init__(self, languages: Optional[List[str]] = None):
def __init__(self, languages: Optional[list[str]] = None):
"""
Initializes the DocumentLanguageClassifier component.
@@ -65,8 +65,8 @@ class DocumentLanguageClassifier:
languages = ["en"]
self.languages = languages
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Classifies the language of each document and adds it to its metadata.
@@ -86,7 +86,7 @@ class DocumentLanguageClassifier:
"In case you want to classify and route a text, please use the TextLanguageRouter."
)
output: Dict[str, List[Document]] = {language: [] for language in self.languages}
output: dict[str, list[Document]] = {language: [] for language in self.languages}
output["unmatched"] = []
for document in documents:
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.lazy_imports import LazyImport
@@ -74,12 +74,12 @@ class TransformersZeroShotDocumentClassifier:
def __init__( # pylint: disable=too-many-positional-arguments
self,
model: str,
labels: List[str],
labels: list[str],
multi_label: bool = False,
classification_field: Optional[str] = None,
device: Optional[ComponentDevice] = None,
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
):
"""
Initializes the TransformersZeroShotDocumentClassifier.
@@ -132,7 +132,7 @@ class TransformersZeroShotDocumentClassifier:
self.huggingface_pipeline_kwargs = huggingface_pipeline_kwargs
self.pipeline: Optional[HfPipeline] = None
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
@@ -147,7 +147,7 @@ class TransformersZeroShotDocumentClassifier:
if self.pipeline is None:
self.pipeline = pipeline(**self.huggingface_pipeline_kwargs)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -169,7 +169,7 @@ class TransformersZeroShotDocumentClassifier:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "TransformersZeroShotDocumentClassifier":
def from_dict(cls, data: dict[str, Any]) -> "TransformersZeroShotDocumentClassifier":
"""
Deserializes the component from a dictionary.
@@ -183,8 +183,8 @@ class TransformersZeroShotDocumentClassifier:
deserialize_hf_model_kwargs(data["init_parameters"]["huggingface_pipeline_kwargs"])
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
def run(self, documents: List[Document], batch_size: int = 1):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document], batch_size: int = 1):
"""
Classifies the documents based on the provided labels and adds them to their metadata.
+6 -6
View File
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, Optional
from typing import Any, Optional
from haystack import component, default_from_dict, default_to_dict
from haystack.lazy_imports import LazyImport
@@ -45,7 +45,7 @@ class OpenAPIConnector:
"""
def __init__(
self, openapi_spec: str, credentials: Optional[Secret] = None, service_kwargs: Optional[Dict[str, Any]] = None
self, openapi_spec: str, credentials: Optional[Secret] = None, service_kwargs: Optional[dict[str, Any]] = None
):
"""
Initialize the OpenAPIConnector with a specification and optional credentials.
@@ -66,7 +66,7 @@ class OpenAPIConnector:
**self.service_kwargs,
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
"""
@@ -78,15 +78,15 @@ class OpenAPIConnector:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAPIConnector":
def from_dict(cls, data: dict[str, Any]) -> "OpenAPIConnector":
"""
Deserialize this component from a dictionary.
"""
deserialize_secrets_inplace(data["init_parameters"], keys=["credentials"])
return default_from_dict(cls, data)
@component.output_types(response=Dict[str, Any])
def run(self, operation_id: str, arguments: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
@component.output_types(response=dict[str, Any])
def run(self, operation_id: str, arguments: Optional[dict[str, Any]] = None) -> dict[str, Any]:
"""
Invokes a REST endpoint specified in the OpenAPI specification.
@@ -5,7 +5,7 @@
import json
from collections import defaultdict
from copy import copy
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import component, default_from_dict, default_to_dict
from haystack.dataclasses import ChatMessage, ChatRole
@@ -25,9 +25,9 @@ with LazyImport("Run 'pip install openapi3'") as openapi_imports:
base_url: str,
*,
data: Optional[Any] = None,
parameters: Optional[Dict[str, Any]] = None,
parameters: Optional[dict[str, Any]] = None,
raw_response: bool = False,
security: Optional[Dict[str, str]] = None,
security: Optional[dict[str, str]] = None,
session: Optional[Any] = None,
verify: Union[bool, str] = True,
) -> Optional[Any]:
@@ -206,13 +206,13 @@ class OpenAPIServiceConnector:
openapi_imports.check()
self.ssl_verify = ssl_verify
@component.output_types(service_response=Dict[str, Any])
@component.output_types(service_response=dict[str, Any])
def run(
self,
messages: List[ChatMessage],
service_openapi_spec: Dict[str, Any],
messages: list[ChatMessage],
service_openapi_spec: dict[str, Any],
service_credentials: Optional[Union[dict, str]] = None,
) -> Dict[str, List[ChatMessage]]:
) -> dict[str, list[ChatMessage]]:
"""
Processes a list of chat messages to invoke a method on an OpenAPI service.
@@ -261,7 +261,7 @@ class OpenAPIServiceConnector:
return {"service_response": response_messages}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -271,7 +271,7 @@ class OpenAPIServiceConnector:
return default_to_dict(self, ssl_verify=self.ssl_verify)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAPIServiceConnector":
def from_dict(cls, data: dict[str, Any]) -> "OpenAPIServiceConnector":
"""
Deserializes the component from a dictionary.
@@ -337,7 +337,7 @@ class OpenAPIServiceConnector:
f"for it. Check the service configuration and credentials."
)
def _invoke_method(self, openapi_service: "OpenAPI", method_invocation_descriptor: Dict[str, Any]) -> Any:
def _invoke_method(self, openapi_service: "OpenAPI", method_invocation_descriptor: dict[str, Any]) -> Any:
"""
Invokes the specified method on the OpenAPI service.
@@ -368,7 +368,7 @@ class OpenAPIServiceConnector:
operation_dict = operation.raw_element
# Pack URL/query parameters under "parameters" key
method_call_params: Dict[str, Dict[str, Any]] = defaultdict(dict)
method_call_params: dict[str, dict[str, Any]] = defaultdict(dict)
parameters = operation_dict.get("parameters", [])
request_body = operation_dict.get("requestBody", {})
+13 -13
View File
@@ -6,7 +6,7 @@ import copy
import os
from collections import defaultdict
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Literal, Optional, Union
import networkx as nx
@@ -108,8 +108,8 @@ class AzureOCRDocumentConverter:
if self.page_layout == "single_column" and self.threshold_y is None:
self.threshold_y = 0.05
@component.output_types(documents=List[Document], raw_azure_response=List[Dict])
def run(self, sources: List[Union[str, Path, ByteStream]], meta: Optional[List[Dict[str, Any]]] = None):
@component.output_types(documents=list[Document], raw_azure_response=list[dict])
def run(self, sources: list[Union[str, Path, ByteStream]], meta: Optional[list[dict[str, Any]]] = None):
"""
Convert a list of files to Documents using Azure's Document Intelligence service.
@@ -129,7 +129,7 @@ class AzureOCRDocumentConverter:
"""
documents = []
azure_output = []
meta_list: List[Dict[str, Any]] = normalize_metadata(meta=meta, sources_count=len(sources))
meta_list: list[dict[str, Any]] = normalize_metadata(meta=meta, sources_count=len(sources))
for source, metadata in zip(sources, meta_list):
try:
bytestream = get_bytestream_from_source(source=source)
@@ -152,7 +152,7 @@ class AzureOCRDocumentConverter:
return {"documents": documents, "raw_azure_response": azure_output}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -173,7 +173,7 @@ class AzureOCRDocumentConverter:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AzureOCRDocumentConverter":
def from_dict(cls, data: dict[str, Any]) -> "AzureOCRDocumentConverter":
"""
Deserializes the component from a dictionary.
@@ -186,7 +186,7 @@ class AzureOCRDocumentConverter:
return default_from_dict(cls, data)
# pylint: disable=line-too-long
def _convert_tables_and_text(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> List[Document]:
def _convert_tables_and_text(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> list[Document]:
"""
Converts the tables and text extracted by Azure's Document Intelligence service into Haystack Documents.
@@ -205,7 +205,7 @@ class AzureOCRDocumentConverter:
docs = [*tables, text]
return docs
def _convert_tables(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> List[Document]:
def _convert_tables(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> list[Document]:
"""
Converts the tables extracted by Azure's Document Intelligence service into Haystack Documents.
@@ -214,7 +214,7 @@ class AzureOCRDocumentConverter:
:returns: List of Documents containing the tables extracted from the AnalyzeResult object.
"""
converted_tables: List[Document] = []
converted_tables: list[Document] = []
if not result.tables:
return converted_tables
@@ -312,7 +312,7 @@ class AzureOCRDocumentConverter:
return converted_tables
def _convert_to_natural_text(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> Document:
def _convert_to_natural_text(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> Document:
"""
This converts the `AnalyzeResult` object into a single document.
@@ -329,7 +329,7 @@ class AzureOCRDocumentConverter:
texts = []
if result.paragraphs:
paragraphs_to_pages: Dict[int, str] = defaultdict(str)
paragraphs_to_pages: dict[int, str] = defaultdict(str)
for paragraph in result.paragraphs:
if paragraph.bounding_regions:
# If paragraph spans multiple pages we group it with the first page number
@@ -357,7 +357,7 @@ class AzureOCRDocumentConverter:
return Document(content=all_text, meta=meta if meta else {})
def _convert_to_single_column_text(
self, result: "AnalyzeResult", meta: Optional[Dict[str, str]], threshold_y: float = 0.05
self, result: "AnalyzeResult", meta: Optional[dict[str, str]], threshold_y: float = 0.05
) -> Document:
"""
This converts the `AnalyzeResult` object into a single Haystack Document.
@@ -446,7 +446,7 @@ class AzureOCRDocumentConverter:
all_text = "\f".join(texts)
return Document(content=all_text, meta=meta if meta else {})
def _collect_table_spans(self, result: "AnalyzeResult") -> Dict:
def _collect_table_spans(self, result: "AnalyzeResult") -> dict:
"""
Collect the spans of all tables by page number.
+4 -4
View File
@@ -5,7 +5,7 @@
import io
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -50,11 +50,11 @@ class CSVToDocument:
self.encoding = encoding
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts a CSV file to a Document.
+9 -9
View File
@@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass
from enum import Enum
from io import StringIO
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -160,7 +160,7 @@ class DOCXToDocument:
self.link_format = DOCXLinkFormat.from_str(link_format) if isinstance(link_format, str) else link_format
self.store_full_path = store_full_path
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -175,7 +175,7 @@ class DOCXToDocument:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DOCXToDocument":
def from_dict(cls, data: dict[str, Any]) -> "DOCXToDocument":
"""
Deserializes the component from a dictionary.
@@ -190,11 +190,11 @@ class DOCXToDocument:
data["init_parameters"]["link_format"] = DOCXLinkFormat.from_str(data["init_parameters"]["link_format"])
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts DOCX files to Documents.
@@ -246,7 +246,7 @@ class DOCXToDocument:
return {"documents": documents}
def _extract_elements(self, document: "DocxDocument") -> List[str]:
def _extract_elements(self, document: "DocxDocument") -> list[str]:
"""
Extracts elements from a DOCX file.
@@ -331,8 +331,8 @@ class DOCXToDocument:
:param table: The DOCX table to convert.
:returns: A Markdown string representation of the table.
"""
markdown: List[str] = []
max_col_widths: List[int] = []
markdown: list[str] = []
max_col_widths: list[int] = []
# Calculate max width for each column
for row in table.rows:
+8 -8
View File
@@ -4,7 +4,7 @@
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -34,7 +34,7 @@ class HTMLToDocument:
```
"""
def __init__(self, extraction_kwargs: Optional[Dict[str, Any]] = None, store_full_path: bool = False):
def __init__(self, extraction_kwargs: Optional[dict[str, Any]] = None, store_full_path: bool = False):
"""
Create an HTMLToDocument component.
@@ -50,7 +50,7 @@ class HTMLToDocument:
self.extraction_kwargs = extraction_kwargs or {}
self.store_full_path = store_full_path
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -60,7 +60,7 @@ class HTMLToDocument:
return default_to_dict(self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HTMLToDocument":
def from_dict(cls, data: dict[str, Any]) -> "HTMLToDocument":
"""
Deserializes the component from a dictionary.
@@ -71,12 +71,12 @@ class HTMLToDocument:
"""
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
extraction_kwargs: Optional[Dict[str, Any]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
extraction_kwargs: Optional[dict[str, Any]] = None,
):
"""
Converts a list of HTML files to Documents.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Dict, List, Literal, Optional, Tuple
from typing import Literal, Optional
from haystack import Document, component, logging
from haystack.components.converters.image.image_utils import (
@@ -68,7 +68,7 @@ class DocumentToImageContent:
file_path_meta_field: str = "file_path",
root_path: Optional[str] = None,
detail: Optional[Literal["auto", "high", "low"]] = None,
size: Optional[Tuple[int, int]] = None,
size: Optional[tuple[int, int]] = None,
):
"""
Initialize the DocumentToImageContent component.
@@ -90,8 +90,8 @@ class DocumentToImageContent:
self.detail = detail
self.size = size
@component.output_types(image_contents=List[Optional[ImageContent]])
def run(self, documents: List[Document]) -> Dict[str, List[Optional[ImageContent]]]:
@component.output_types(image_contents=list[Optional[ImageContent]])
def run(self, documents: list[Document]) -> dict[str, list[Optional[ImageContent]]]:
"""
Convert documents with image or PDF sources into ImageContent objects.
@@ -117,9 +117,9 @@ class DocumentToImageContent:
documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
)
image_contents: List[Optional[ImageContent]] = [None] * len(documents)
image_contents: list[Optional[ImageContent]] = [None] * len(documents)
pdf_page_infos: List[_PDFPageInfo] = []
pdf_page_infos: list[_PDFPageInfo] = []
for doc_idx, image_source_info in enumerate(images_source_info):
mime_type = image_source_info["mime_type"]
@@ -142,7 +142,7 @@ class DocumentToImageContent:
)
# efficiently convert PDF pages to images: each PDF is opened and processed only once
pdf_page_infos_by_doc_idx: Dict[int, _PDFPageInfo] = {
pdf_page_infos_by_doc_idx: dict[int, _PDFPageInfo] = {
pdf_page_info["doc_idx"]: pdf_page_info for pdf_page_info in pdf_page_infos
}
pdf_images_by_doc_idx = _batch_convert_pdf_pages_to_images(
@@ -4,7 +4,7 @@
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -52,13 +52,13 @@ class ImageFileToDocument:
"""
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
*,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
) -> Dict[str, List[Document]]:
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
) -> dict[str, list[Document]]:
"""
Convert image files into empty Document objects with metadata.
@@ -4,7 +4,7 @@
import mimetypes
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from typing import Any, Literal, Optional, Union
from haystack import component, logging
from haystack.components.converters.image.image_utils import _encode_image_to_base64
@@ -51,7 +51,7 @@ class ImageFileToImageContent:
"""
def __init__(
self, *, detail: Optional[Literal["auto", "high", "low"]] = None, size: Optional[Tuple[int, int]] = None
self, *, detail: Optional[Literal["auto", "high", "low"]] = None, size: Optional[tuple[int, int]] = None
):
"""
Create the ImageFileToImageContent component.
@@ -68,15 +68,15 @@ class ImageFileToImageContent:
if self.size is not None:
pillow_import.check()
@component.output_types(image_contents=List[ImageContent])
@component.output_types(image_contents=list[ImageContent])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
*,
detail: Optional[Literal["auto", "high", "low"]] = None,
size: Optional[Tuple[int, int]] = None,
) -> Dict[str, List[ImageContent]]:
size: Optional[tuple[int, int]] = None,
) -> dict[str, list[ImageContent]]:
"""
Converts files to ImageContent objects.
@@ -7,7 +7,7 @@ import mimetypes
from collections import defaultdict
from io import BytesIO
from pathlib import Path
from typing import Dict, List, Optional, Tuple, TypedDict, Union
from typing import Optional, TypedDict, Union
from typing_extensions import NotRequired
@@ -29,8 +29,8 @@ logger = logging.getLogger(__name__)
def _encode_image_to_base64(
bytestream: ByteStream, size: Optional[Tuple[int, int]] = None
) -> Tuple[Optional[str], str]:
bytestream: ByteStream, size: Optional[tuple[int, int]] = None
) -> tuple[Optional[str], str]:
"""
Encode an image from a ByteStream into a base64-encoded string.
@@ -118,9 +118,9 @@ def _convert_pdf_to_images(
*,
bytestream: ByteStream,
return_base64: bool = False,
page_range: Optional[List[int]] = None,
size: Optional[Tuple[int, int]] = None,
) -> Union[List[Tuple[int, "Image"]], List[Tuple[int, str]]]:
page_range: Optional[list[int]] = None,
size: Optional[tuple[int, int]] = None,
) -> Union[list[tuple[int, "Image"]], list[tuple[int, str]]]:
"""
Convert a PDF file into a list of PIL Image objects or base64-encoded images.
@@ -225,8 +225,8 @@ class _ImageSourceInfo(TypedDict):
def _extract_image_sources_info(
documents: List[Document], file_path_meta_field: str, root_path: str
) -> List[_ImageSourceInfo]:
documents: list[Document], file_path_meta_field: str, root_path: str
) -> list[_ImageSourceInfo]:
"""
Extracts the image source information from the documents.
@@ -240,7 +240,7 @@ def _extract_image_sources_info(
:raises ValueError: If the document is missing the file_path_meta_field key in its metadata, the file path is
invalid, the MIME type is not supported, or the page number is missing for a PDF document.
"""
images_source_info: List[_ImageSourceInfo] = []
images_source_info: list[_ImageSourceInfo] = []
for doc in documents:
file_path = doc.meta.get(file_path_meta_field)
if file_path is None:
@@ -289,8 +289,8 @@ class _PDFPageInfo(TypedDict):
def _batch_convert_pdf_pages_to_images(
*, pdf_page_infos: List[_PDFPageInfo], return_base64: bool = False, size: Optional[Tuple[int, int]] = None
) -> Union[Dict[int, str], Dict[int, "Image"]]:
*, pdf_page_infos: list[_PDFPageInfo], return_base64: bool = False, size: Optional[tuple[int, int]] = None
) -> Union[dict[int, str], dict[int, "Image"]]:
"""
Converts selected PDF pages to images, returning a mapping from document indices to images (PIL or base64).
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from typing import Any, Literal, Optional, Union
from haystack import component, logging
from haystack.components.converters.image.image_utils import _convert_pdf_to_images, pillow_import, pypdfium2_import
@@ -43,8 +43,8 @@ class PDFToImageContent:
self,
*,
detail: Optional[Literal["auto", "high", "low"]] = None,
size: Optional[Tuple[int, int]] = None,
page_range: Optional[List[Union[str, int]]] = None,
size: Optional[tuple[int, int]] = None,
page_range: Optional[list[Union[str, int]]] = None,
):
"""
Create the PDFToImageContent component.
@@ -66,16 +66,16 @@ class PDFToImageContent:
pypdfium2_import.check()
pillow_import.check()
@component.output_types(image_contents=List[ImageContent])
@component.output_types(image_contents=list[ImageContent])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
*,
detail: Optional[Literal["auto", "high", "low"]] = None,
size: Optional[Tuple[int, int]] = None,
page_range: Optional[List[Union[str, int]]] = None,
) -> Dict[str, List[ImageContent]]:
size: Optional[tuple[int, int]] = None,
page_range: Optional[list[Union[str, int]]] = None,
) -> dict[str, list[ImageContent]]:
"""
Converts files to ImageContent objects.
+8 -8
View File
@@ -5,7 +5,7 @@
import json
import os
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union
from typing import Any, Literal, Optional, Union
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -93,7 +93,7 @@ class JSONConverter:
self,
jq_schema: Optional[str] = None,
content_key: Optional[str] = None,
extra_meta_fields: Optional[Union[Set[str], Literal["*"]]] = None,
extra_meta_fields: Optional[Union[set[str], Literal["*"]]] = None,
store_full_path: bool = False,
):
"""
@@ -149,7 +149,7 @@ class JSONConverter:
msg = "No `jq_schema` nor `content_key` specified. Set either or both to extract data."
raise ValueError(msg)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -165,7 +165,7 @@ class JSONConverter:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "JSONConverter":
def from_dict(cls, data: dict[str, Any]) -> "JSONConverter":
"""
Deserializes the component from a dictionary.
@@ -176,7 +176,7 @@ class JSONConverter:
"""
return default_from_dict(cls, data)
def _get_content_and_meta(self, source: ByteStream) -> List[Tuple[str, Dict[str, Any]]]:
def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, Any]]]:
"""
Utility function to extract text and metadata from a JSON file.
@@ -246,11 +246,11 @@ class JSONConverter:
return result
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts a list of JSON files to documents.
+4 -4
View File
@@ -4,7 +4,7 @@
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from tqdm import tqdm
@@ -57,11 +57,11 @@ class MarkdownToDocument:
self.progress_bar = progress_bar
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts a list of Markdown files to Documents.
+6 -6
View File
@@ -5,7 +5,7 @@
import io
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -78,7 +78,7 @@ class MSGToDocument:
recip_str += f"{recip.email_address}"
return recip_str
def _convert(self, file_content: io.BytesIO) -> Tuple[str, List[ByteStream]]:
def _convert(self, file_content: io.BytesIO) -> tuple[str, list[ByteStream]]:
"""
Converts the MSG file content into text and extracts any attachments.
@@ -130,12 +130,12 @@ class MSGToDocument:
return txt, attachments
@component.output_types(documents=List[Document], attachments=List[ByteStream])
@component.output_types(documents=list[Document], attachments=list[ByteStream])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
) -> Dict[str, Union[List[Document], List[ByteStream]]]:
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
) -> dict[str, Union[list[Document], list[ByteStream]]]:
"""
Converts MSG files to Documents.
@@ -5,7 +5,7 @@
import json
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
import yaml
@@ -52,8 +52,8 @@ class OpenAPIServiceToFunctions:
"""
openapi_imports.check()
@component.output_types(functions=List[Dict[str, Any]], openapi_specs=List[Dict[str, Any]])
def run(self, sources: List[Union[str, Path, ByteStream]]) -> Dict[str, Any]:
@component.output_types(functions=list[dict[str, Any]], openapi_specs=list[dict[str, Any]])
def run(self, sources: list[Union[str, Path, ByteStream]]) -> dict[str, Any]:
"""
Converts OpenAPI definitions in OpenAI function calling format.
@@ -70,7 +70,7 @@ class OpenAPIServiceToFunctions:
:raises ValueError:
If the source type is not recognized or no functions are found in the OpenAPI definitions.
"""
all_extracted_fc_definitions: List[Dict[str, Any]] = []
all_extracted_fc_definitions: list[dict[str, Any]] = []
all_openapi_specs = []
for source in sources:
openapi_spec_content = None
@@ -101,7 +101,7 @@ class OpenAPIServiceToFunctions:
if openapi_spec_content:
try:
service_openapi_spec = self._parse_openapi_spec(openapi_spec_content)
functions: List[Dict[str, Any]] = self._openapi_to_functions(service_openapi_spec)
functions: list[dict[str, Any]] = self._openapi_to_functions(service_openapi_spec)
all_extracted_fc_definitions.extend(functions)
all_openapi_specs.append(service_openapi_spec)
except Exception as e:
@@ -114,7 +114,7 @@ class OpenAPIServiceToFunctions:
return {"functions": all_extracted_fc_definitions, "openapi_specs": all_openapi_specs}
def _openapi_to_functions(self, service_openapi_spec: Dict[str, Any]) -> List[Dict[str, Any]]:
def _openapi_to_functions(self, service_openapi_spec: dict[str, Any]) -> list[dict[str, Any]]:
"""
OpenAPI to OpenAI function conversion.
@@ -122,10 +122,10 @@ class OpenAPIServiceToFunctions:
suitable for OpenAI function calling.
:param service_openapi_spec: The OpenAPI specification from which functions are to be extracted.
:type service_openapi_spec: Dict[str, Any]
:type service_openapi_spec: dict[str, Any]
:return: A list of dictionaries, each representing a function. Each dictionary includes the function's
name, description, and a schema of its parameters.
:rtype: List[Dict[str, Any]]
:rtype: list[dict[str, Any]]
"""
# Doesn't enforce rigid spec validation because that would require a lot of dependencies
@@ -142,7 +142,7 @@ class OpenAPIServiceToFunctions:
f"at least {OpenAPIServiceToFunctions.MIN_REQUIRED_OPENAPI_SPEC_VERSION}."
)
functions: List[Dict[str, Any]] = []
functions: list[dict[str, Any]] = []
for paths in service_openapi_spec["paths"].values():
for path_spec in paths.values():
function_dict = self._parse_endpoint_spec(path_spec)
@@ -150,7 +150,7 @@ class OpenAPIServiceToFunctions:
functions.append(function_dict)
return functions
def _parse_endpoint_spec(self, resolved_spec: Dict[str, Any]) -> Optional[Dict[str, Any]]:
def _parse_endpoint_spec(self, resolved_spec: dict[str, Any]) -> Optional[dict[str, Any]]:
if not isinstance(resolved_spec, dict):
logger.warning("Invalid OpenAPI spec format provided. Could not extract function.")
return {}
@@ -158,7 +158,7 @@ class OpenAPIServiceToFunctions:
function_name = resolved_spec.get("operationId")
description = resolved_spec.get("description") or resolved_spec.get("summary", "")
schema: Dict[str, Any] = {"type": "object", "properties": {}}
schema: dict[str, Any] = {"type": "object", "properties": {}}
# requestBody section
req_body_schema = (
@@ -191,8 +191,8 @@ class OpenAPIServiceToFunctions:
return {}
def _parse_property_attributes(
self, property_schema: Dict[str, Any], include_attributes: Optional[List[str]] = None
) -> Dict[str, Any]:
self, property_schema: dict[str, Any], include_attributes: Optional[list[str]] = None
) -> dict[str, Any]:
"""
Parses the attributes of a property schema.
@@ -229,7 +229,7 @@ class OpenAPIServiceToFunctions:
return parsed_schema
def _parse_openapi_spec(self, content: str) -> Dict[str, Any]:
def _parse_openapi_spec(self, content: str) -> dict[str, Any]:
"""
Parses OpenAPI specification content, supporting both JSON and YAML formats.
@@ -4,7 +4,7 @@
import ast
import contextlib
from typing import Any, Callable, Dict, Optional, Set
from typing import Any, Callable, Optional
import jinja2.runtime
from jinja2 import Environment, TemplateSyntaxError, meta
@@ -44,7 +44,7 @@ class OutputAdapter:
self,
template: str,
output_type: TypeAlias,
custom_filters: Optional[Dict[str, Callable]] = None,
custom_filters: Optional[dict[str, Callable]] = None,
unsafe: bool = False,
):
"""
@@ -68,7 +68,7 @@ class OutputAdapter:
This should only be used if you trust the source of the template as it can be lead to remote code execution.
"""
self.custom_filters = {**(custom_filters or {})}
input_types: Set[str] = set()
input_types: set[str] = set()
self._unsafe = unsafe
@@ -137,7 +137,7 @@ class OutputAdapter:
raise OutputAdaptationException(f"Error adapting {self.template} with {kwargs}: {e}") from e
return adapted_outputs
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -154,7 +154,7 @@ class OutputAdapter:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OutputAdapter":
def from_dict(cls, data: dict[str, Any]) -> "OutputAdapter":
"""
Deserializes the component from a dictionary.
@@ -174,7 +174,7 @@ class OutputAdapter:
}
return default_from_dict(cls, data)
def _extract_variables(self, env: Environment) -> Set[str]:
def _extract_variables(self, env: Environment) -> set[str]:
"""
Extracts all variables from a list of Jinja template strings.
+5 -5
View File
@@ -6,7 +6,7 @@ import io
import os
import re
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Union
from typing import Any, Iterator, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -130,7 +130,7 @@ class PDFMinerToDocument:
return delimited_pages
def detect_undecoded_cid_characters(self, text: str) -> Dict[str, Any]:
def detect_undecoded_cid_characters(self, text: str) -> dict[str, Any]:
"""
Look for character sequences of CID, i.e.: characters that haven't been properly decoded from their CID format.
@@ -156,11 +156,11 @@ class PDFMinerToDocument:
return {"total_chars": total_chars, "cid_chars": cid_chars, "percentage": round(percentage, 2)}
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts PDF files to Documents.
+4 -4
View File
@@ -5,7 +5,7 @@
import io
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -62,11 +62,11 @@ class PPTXToDocument:
text = "\f".join(text_all_slides)
return text
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts PPTX files to Documents.
+4 -4
View File
@@ -6,7 +6,7 @@ import io
import os
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -171,11 +171,11 @@ class PyPDFToDocument:
text = "\f".join(texts)
return text
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts PDF files to documents.
+7 -7
View File
@@ -6,7 +6,7 @@ import io
import os
from html.parser import HTMLParser
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -19,7 +19,7 @@ with LazyImport("Run 'pip install tika'") as tika_import:
logger = logging.getLogger(__name__)
class XHTMLParser(HTMLParser):
class XHTMLParser(HTMLParser): # pylint: disable=abstract-method
"""
Custom parser to extract pages from Tika XHTML content.
"""
@@ -28,9 +28,9 @@ class XHTMLParser(HTMLParser):
super().__init__()
self.ingest = True
self.page = ""
self.pages: List[str] = []
self.pages: list[str] = []
def handle_starttag(self, tag: str, attrs: List[tuple]):
def handle_starttag(self, tag: str, attrs: list[tuple]):
"""Identify the start of a page div."""
if tag == "div" and any(attr == "class" and value == "page" for attr, value in attrs):
self.ingest = True
@@ -88,11 +88,11 @@ class TikaDocumentConverter:
self.tika_url = tika_url
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts files to Documents.
+4 -4
View File
@@ -4,7 +4,7 @@
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -50,11 +50,11 @@ class TextFileToDocument:
self.encoding = encoding
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
):
"""
Converts text files to documents.
+3 -3
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack.dataclasses import ByteStream
@@ -30,8 +30,8 @@ def get_bytestream_from_source(source: Union[str, Path, ByteStream], guess_mime_
def normalize_metadata(
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]], sources_count: int
) -> List[Dict[str, Any]]:
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]], sources_count: int
) -> list[dict[str, Any]]:
"""
Normalize the metadata input for a converter.
+10 -10
View File
@@ -5,7 +5,7 @@
import io
import os
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from typing import Any, Literal, Optional, Union
from haystack import Document, component, logging
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
@@ -46,9 +46,9 @@ class XLSXToDocument:
def __init__(
self,
table_format: Literal["csv", "markdown"] = "csv",
sheet_name: Union[str, int, List[Union[str, int]], None] = None,
read_excel_kwargs: Optional[Dict[str, Any]] = None,
table_format_kwargs: Optional[Dict[str, Any]] = None,
sheet_name: Union[str, int, list[Union[str, int]], None] = None,
read_excel_kwargs: Optional[dict[str, Any]] = None,
table_format_kwargs: Optional[dict[str, Any]] = None,
*,
store_full_path: bool = False,
):
@@ -79,12 +79,12 @@ class XLSXToDocument:
self.table_format_kwargs = table_format_kwargs or {}
self.store_full_path = store_full_path
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
sources: List[Union[str, Path, ByteStream]],
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
) -> Dict[str, List[Document]]:
sources: list[Union[str, Path, ByteStream]],
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
) -> dict[str, list[Document]]:
"""
Converts a XLSX file to a Document.
@@ -136,7 +136,7 @@ class XLSXToDocument:
return {"documents": documents}
@staticmethod
def _generate_excel_column_names(n_cols: int) -> List[str]:
def _generate_excel_column_names(n_cols: int) -> list[str]:
result = []
for i in range(n_cols):
col_name = ""
@@ -147,7 +147,7 @@ class XLSXToDocument:
result.append(col_name)
return result
def _extract_tables(self, bytestream: ByteStream) -> Tuple[List[str], List[Dict]]:
def _extract_tables(self, bytestream: ByteStream) -> tuple[list[str], list[dict]]:
"""
Extract tables from a Excel file.
"""
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
@@ -51,14 +51,14 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
suffix: str = "",
batch_size: int = 32,
progress_bar: bool = True,
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
*,
default_headers: Optional[Dict[str, str]] = None,
default_headers: Optional[dict[str, str]] = None,
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
raise_on_failure: bool = False,
):
"""
@@ -146,7 +146,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
self.http_client_kwargs = http_client_kwargs
self.raise_on_failure = raise_on_failure
client_args: Dict[str, Any] = {
client_args: dict[str, Any] = {
"api_version": api_version,
"azure_endpoint": azure_endpoint,
"azure_deployment": azure_deployment,
@@ -166,7 +166,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_args
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -200,7 +200,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIDocumentEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIDocumentEmbedder":
"""
Deserializes the component from a dictionary.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, Optional
from typing import Any, Optional
from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
@@ -50,9 +50,9 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
prefix: str = "",
suffix: str = "",
*,
default_headers: Optional[Dict[str, str]] = None,
default_headers: Optional[dict[str, str]] = None,
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Creates an AzureOpenAITextEmbedder component.
@@ -127,7 +127,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
self.azure_ad_token_provider = azure_ad_token_provider
self.http_client_kwargs = http_client_kwargs
client_kwargs: Dict[str, Any] = {
client_kwargs: dict[str, Any] = {
"api_version": api_version,
"azure_endpoint": azure_endpoint,
"azure_deployment": azure_deployment,
@@ -147,7 +147,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -176,7 +176,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAITextEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAITextEmbedder":
"""
Deserializes the component from a dictionary.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Literal, Optional, Union
from haystack.lazy_imports import LazyImport
from haystack.utils.auth import Secret
@@ -19,7 +19,7 @@ class _SentenceTransformersEmbeddingBackendFactory:
Factory class to create instances of Sentence Transformers embedding backends.
"""
_instances: Dict[str, "_SentenceTransformersEmbeddingBackend"] = {}
_instances: dict[str, "_SentenceTransformersEmbeddingBackend"] = {}
@staticmethod
def get_embedding_backend( # pylint: disable=too-many-positional-arguments
@@ -29,9 +29,9 @@ class _SentenceTransformersEmbeddingBackendFactory:
trust_remote_code: bool = False,
local_files_only: bool = False,
truncate_dim: Optional[int] = None,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
):
embedding_backend_id = f"{model}{device}{auth_token}{truncate_dim}{backend}"
@@ -67,9 +67,9 @@ class _SentenceTransformersEmbeddingBackend:
trust_remote_code: bool = False,
local_files_only: bool = False,
truncate_dim: Optional[int] = None,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
):
sentence_transformers_import.check()
@@ -87,7 +87,7 @@ class _SentenceTransformersEmbeddingBackend:
backend=backend,
)
def embed(self, data: Union[List[str], List["Image"]], **kwargs: Any) -> List[List[float]]:
def embed(self, data: Union[list[str], list["Image"]], **kwargs: Any) -> list[list[float]]:
# Sentence Transformers encode can work with Images, but the type hint does not reflect that
# https://sbert.net/examples/sentence_transformer/applications/image-search
embeddings = self.model.encode(data, **kwargs).tolist() # type: ignore[arg-type]
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Optional, Union
from tqdm import tqdm
from tqdm.asyncio import tqdm as async_tqdm
@@ -92,7 +92,7 @@ class HuggingFaceAPIDocumentEmbedder:
def __init__(
self,
api_type: Union[HFEmbeddingAPIType, str],
api_params: Dict[str, str],
api_params: dict[str, str],
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
prefix: str = "",
suffix: str = "",
@@ -100,7 +100,7 @@ class HuggingFaceAPIDocumentEmbedder:
normalize: Optional[bool] = False,
batch_size: int = 32,
progress_bar: bool = True,
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
): # pylint: disable=too-many-positional-arguments
"""
@@ -168,7 +168,7 @@ class HuggingFaceAPIDocumentEmbedder:
msg = f"Unknown api_type {api_type}"
raise ValueError(msg)
client_args: Dict[str, Any] = {"model": model_or_url, "token": token.resolve_value() if token else None}
client_args: dict[str, Any] = {"model": model_or_url, "token": token.resolve_value() if token else None}
self.api_type = api_type
self.api_params = api_params
@@ -184,7 +184,7 @@ class HuggingFaceAPIDocumentEmbedder:
self._client = InferenceClient(**client_args)
self._async_client = AsyncInferenceClient(**client_args)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -207,7 +207,7 @@ class HuggingFaceAPIDocumentEmbedder:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIDocumentEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIDocumentEmbedder":
"""
Deserializes the component from a dictionary.
@@ -219,7 +219,7 @@ class HuggingFaceAPIDocumentEmbedder:
deserialize_secrets_inplace(data["init_parameters"], keys=["token"])
return default_from_dict(cls, data)
def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
"""
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
"""
@@ -239,7 +239,7 @@ class HuggingFaceAPIDocumentEmbedder:
@staticmethod
def _adjust_api_parameters(
truncate: Optional[bool], normalize: Optional[bool], api_type: HFEmbeddingAPIType
) -> Tuple[Optional[bool], Optional[bool]]:
) -> tuple[Optional[bool], Optional[bool]]:
"""
Adjust the truncate and normalize parameters based on the API type.
"""
@@ -254,13 +254,13 @@ class HuggingFaceAPIDocumentEmbedder:
normalize = None
return truncate, normalize
def _embed_batch(self, texts_to_embed: List[str], batch_size: int) -> List[List[float]]:
def _embed_batch(self, texts_to_embed: list[str], batch_size: int) -> list[list[float]]:
"""
Embed a list of texts in batches.
"""
truncate, normalize = self._adjust_api_parameters(self.truncate, self.normalize, self.api_type)
all_embeddings: List = []
all_embeddings: list = []
for i in tqdm(
range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
):
@@ -280,13 +280,13 @@ class HuggingFaceAPIDocumentEmbedder:
return all_embeddings
async def _embed_batch_async(self, texts_to_embed: List[str], batch_size: int) -> List[List[float]]:
async def _embed_batch_async(self, texts_to_embed: list[str], batch_size: int) -> list[list[float]]:
"""
Embed a list of texts in batches asynchronously.
"""
truncate, normalize = self._adjust_api_parameters(self.truncate, self.normalize, self.api_type)
all_embeddings: List = []
all_embeddings: list = []
for i in async_tqdm(
range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
):
@@ -306,8 +306,8 @@ class HuggingFaceAPIDocumentEmbedder:
return all_embeddings
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Embeds a list of documents.
@@ -333,8 +333,8 @@ class HuggingFaceAPIDocumentEmbedder:
return {"documents": documents}
@component.output_types(documents=List[Document])
async def run_async(self, documents: List[Document]):
@component.output_types(documents=list[Document])
async def run_async(self, documents: list[Document]):
"""
Embeds a list of documents asynchronously.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.lazy_imports import LazyImport
@@ -75,7 +75,7 @@ class HuggingFaceAPITextEmbedder:
def __init__(
self,
api_type: Union[HFEmbeddingAPIType, str],
api_params: Dict[str, str],
api_params: dict[str, str],
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
prefix: str = "",
suffix: str = "",
@@ -171,7 +171,7 @@ class HuggingFaceAPITextEmbedder:
return text_to_embed, truncate, normalize
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -190,7 +190,7 @@ class HuggingFaceAPITextEmbedder:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPITextEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPITextEmbedder":
"""
Deserializes the component from a dictionary.
@@ -202,7 +202,7 @@ class HuggingFaceAPITextEmbedder:
deserialize_secrets_inplace(data["init_parameters"], keys=["token"])
return default_from_dict(cls, data)
@component.output_types(embedding=List[float])
@component.output_types(embedding=list[float])
def run(self, text: str):
"""
Embeds a single string.
@@ -230,7 +230,7 @@ class HuggingFaceAPITextEmbedder:
return {"embedding": embedding}
@component.output_types(embedding=List[float])
@component.output_types(embedding=list[float])
async def run_async(self, text: str):
"""
Embeds a single string asynchronously.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from copy import copy
from typing import Any, Dict, List, Literal, Optional
from typing import Any, Literal, Optional
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.components.converters.image.image_utils import (
@@ -70,11 +70,11 @@ class SentenceTransformersDocumentImageEmbedder:
normalize_embeddings: bool = False,
trust_remote_code: bool = False,
local_files_only: bool = False,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
encode_kwargs: Optional[Dict[str, Any]] = None,
encode_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
) -> None:
"""
@@ -152,7 +152,7 @@ class SentenceTransformersDocumentImageEmbedder:
self.backend = backend
self._embedding_backend: Optional[_SentenceTransformersEmbeddingBackend] = None
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -183,7 +183,7 @@ class SentenceTransformersDocumentImageEmbedder:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDocumentImageEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDocumentImageEmbedder":
"""
Deserializes the component from a dictionary.
@@ -219,8 +219,8 @@ class SentenceTransformersDocumentImageEmbedder:
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
self._embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
"""
Embed a list of documents.
@@ -243,8 +243,8 @@ class SentenceTransformersDocumentImageEmbedder:
documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
)
images_to_embed: List = [None] * len(documents)
pdf_page_infos: List[_PDFPageInfo] = []
images_to_embed: list = [None] * len(documents)
pdf_page_infos: list[_PDFPageInfo] = []
for doc_idx, image_source_info in enumerate(images_source_info):
if image_source_info["mime_type"] == "application/pdf":
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Optional
from more_itertools import batched
from openai import APIError, AsyncOpenAI, OpenAI
@@ -50,11 +50,11 @@ class OpenAIDocumentEmbedder:
suffix: str = "",
batch_size: int = 32,
progress_bar: bool = True,
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
*,
raise_on_failure: bool = False,
):
@@ -127,7 +127,7 @@ class OpenAIDocumentEmbedder:
if max_retries is None:
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
client_kwargs: Dict[str, Any] = {
client_kwargs: dict[str, Any] = {
"api_key": api_key.resolve_value(),
"organization": organization,
"base_url": api_base_url,
@@ -140,13 +140,13 @@ class OpenAIDocumentEmbedder:
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
)
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -173,7 +173,7 @@ class OpenAIDocumentEmbedder:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIDocumentEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "OpenAIDocumentEmbedder":
"""
Deserializes the component from a dictionary.
@@ -185,7 +185,7 @@ class OpenAIDocumentEmbedder:
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
return default_from_dict(cls, data)
def _prepare_texts_to_embed(self, documents: List[Document]) -> Dict[str, str]:
def _prepare_texts_to_embed(self, documents: list[Document]) -> dict[str, str]:
"""
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
"""
@@ -202,18 +202,18 @@ class OpenAIDocumentEmbedder:
return texts_to_embed
def _embed_batch(
self, texts_to_embed: Dict[str, str], batch_size: int
) -> Tuple[Dict[str, List[float]], Dict[str, Any]]:
self, texts_to_embed: dict[str, str], batch_size: int
) -> tuple[dict[str, list[float]], dict[str, Any]]:
"""
Embed a list of texts in batches.
"""
doc_ids_to_embeddings: Dict[str, List[float]] = {}
meta: Dict[str, Any] = {}
doc_ids_to_embeddings: dict[str, list[float]] = {}
meta: dict[str, Any] = {}
for batch in tqdm(
batched(texts_to_embed.items(), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
):
args: Dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch], "encoding_format": "float"}
args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch], "encoding_format": "float"}
if self.dimensions is not None:
args["dimensions"] = self.dimensions
@@ -242,21 +242,21 @@ class OpenAIDocumentEmbedder:
return doc_ids_to_embeddings, meta
async def _embed_batch_async(
self, texts_to_embed: Dict[str, str], batch_size: int
) -> Tuple[Dict[str, List[float]], Dict[str, Any]]:
self, texts_to_embed: dict[str, str], batch_size: int
) -> tuple[dict[str, list[float]], dict[str, Any]]:
"""
Embed a list of texts in batches asynchronously.
"""
doc_ids_to_embeddings: Dict[str, List[float]] = {}
meta: Dict[str, Any] = {}
doc_ids_to_embeddings: dict[str, list[float]] = {}
meta: dict[str, Any] = {}
batches = list(batched(texts_to_embed.items(), batch_size))
if self.progress_bar:
batches = async_tqdm(batches, desc="Calculating embeddings")
for batch in batches:
args: Dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch]}
args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch]}
if self.dimensions is not None:
args["dimensions"] = self.dimensions
@@ -284,8 +284,8 @@ class OpenAIDocumentEmbedder:
return doc_ids_to_embeddings, meta
@component.output_types(documents=List[Document], meta=Dict[str, Any])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document], meta=dict[str, Any])
def run(self, documents: list[Document]):
"""
Embeds a list of documents.
@@ -313,8 +313,8 @@ class OpenAIDocumentEmbedder:
return {"documents": list(doc_id_to_document.values()), "meta": meta}
@component.output_types(documents=List[Document], meta=Dict[str, Any])
async def run_async(self, documents: List[Document]):
@component.output_types(documents=list[Document], meta=dict[str, Any])
async def run_async(self, documents: list[Document]):
"""
Embeds a list of documents asynchronously.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from openai import AsyncOpenAI, OpenAI
from openai.types import CreateEmbeddingResponse
@@ -48,7 +48,7 @@ class OpenAITextEmbedder:
suffix: str = "",
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Creates an OpenAITextEmbedder component.
@@ -103,7 +103,7 @@ class OpenAITextEmbedder:
if max_retries is None:
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
client_kwargs: Dict[str, Any] = {
client_kwargs: dict[str, Any] = {
"api_key": api_key.resolve_value(),
"organization": organization,
"base_url": api_base_url,
@@ -116,13 +116,13 @@ class OpenAITextEmbedder:
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
)
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -144,7 +144,7 @@ class OpenAITextEmbedder:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAITextEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "OpenAITextEmbedder":
"""
Deserializes the component from a dictionary.
@@ -156,7 +156,7 @@ class OpenAITextEmbedder:
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
return default_from_dict(cls, data)
def _prepare_input(self, text: str) -> Dict[str, Any]:
def _prepare_input(self, text: str) -> dict[str, Any]:
if not isinstance(text, str):
raise TypeError(
"OpenAITextEmbedder expects a string as an input."
@@ -165,15 +165,15 @@ class OpenAITextEmbedder:
text_to_embed = self.prefix + text + self.suffix
kwargs: Dict[str, Any] = {"model": self.model, "input": text_to_embed, "encoding_format": "float"}
kwargs: dict[str, Any] = {"model": self.model, "input": text_to_embed, "encoding_format": "float"}
if self.dimensions is not None:
kwargs["dimensions"] = self.dimensions
return kwargs
def _prepare_output(self, result: CreateEmbeddingResponse) -> Dict[str, Any]:
def _prepare_output(self, result: CreateEmbeddingResponse) -> dict[str, Any]:
return {"embedding": result.data[0].embedding, "meta": {"model": result.model, "usage": dict(result.usage)}}
@component.output_types(embedding=List[float], meta=Dict[str, Any])
@component.output_types(embedding=list[float], meta=dict[str, Any])
def run(self, text: str):
"""
Embeds a single string.
@@ -190,7 +190,7 @@ class OpenAITextEmbedder:
response = self.client.embeddings.create(**create_kwargs)
return self._prepare_output(result=response)
@component.output_types(embedding=List[float], meta=Dict[str, Any])
@component.output_types(embedding=list[float], meta=dict[str, Any])
async def run_async(self, text: str):
"""
Asynchronously embed a single string.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Optional
from typing import Any, Literal, Optional
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.components.embedders.backends.sentence_transformers_backend import (
@@ -49,16 +49,16 @@ class SentenceTransformersDocumentEmbedder:
batch_size: int = 32,
progress_bar: bool = True,
normalize_embeddings: bool = False,
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
trust_remote_code: bool = False,
local_files_only: bool = False,
truncate_dim: Optional[int] = None,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
encode_kwargs: Optional[Dict[str, Any]] = None,
encode_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
):
"""
@@ -141,13 +141,13 @@ class SentenceTransformersDocumentEmbedder:
self.precision = precision
self.backend = backend
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -181,7 +181,7 @@ class SentenceTransformersDocumentEmbedder:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDocumentEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDocumentEmbedder":
"""
Deserializes the component from a dictionary.
@@ -218,8 +218,8 @@ class SentenceTransformersDocumentEmbedder:
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
self.embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Embed a list of documents.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Optional
from typing import Any, Literal, Optional
from haystack import component, default_from_dict, default_to_dict
from haystack.components.embedders.backends.sentence_transformers_backend import (
@@ -48,11 +48,11 @@ class SentenceTransformersTextEmbedder:
trust_remote_code: bool = False,
local_files_only: bool = False,
truncate_dim: Optional[int] = None,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
encode_kwargs: Optional[Dict[str, Any]] = None,
encode_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
):
"""
@@ -129,13 +129,13 @@ class SentenceTransformersTextEmbedder:
self.precision = precision
self.backend = backend
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -167,7 +167,7 @@ class SentenceTransformersTextEmbedder:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersTextEmbedder":
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersTextEmbedder":
"""
Deserializes the component from a dictionary.
@@ -204,7 +204,7 @@ class SentenceTransformersTextEmbedder:
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
self.embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
@component.output_types(embedding=List[float])
@component.output_types(embedding=list[float])
def run(self, text: str):
"""
Embed a single string.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, Protocol
from typing import Any, Protocol
# See https://github.com/pylint-dev/pylint/issues/9319.
# pylint: disable=unnecessary-ellipsis
@@ -13,7 +13,7 @@ class TextEmbedder(Protocol):
Protocol for Text Embedders.
"""
def run(self, text: str) -> Dict[str, Any]:
def run(self, text: str) -> dict[str, Any]:
"""
Generate embeddings for the input text.
@@ -24,7 +24,7 @@ class TextEmbedder(Protocol):
The input text to be embedded.
:returns:
A dictionary containing the keys:
- 'embedding', which is expected to be a List[float] representing the embedding.
- 'embedding', which is expected to be a list[float] representing the embedding.
- any optional keys such as 'metadata'.
"""
...
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List
from typing import Any
from haystack.core.component import component
@@ -35,8 +35,8 @@ class AnswerExactMatchEvaluator:
```
"""
@component.output_types(individual_scores=List[int], score=float)
def run(self, ground_truth_answers: List[str], predicted_answers: List[str]) -> Dict[str, Any]:
@component.output_types(individual_scores=list[int], score=float)
def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
"""
Run the AnswerExactMatchEvaluator on the given inputs.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from statistics import mean
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from haystack import component, default_from_dict, default_to_dict
from haystack.components.evaluators.llm_evaluator import LLMEvaluator
@@ -99,7 +99,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
def __init__( # pylint: disable=too-many-positional-arguments
self,
examples: Optional[List[Dict[str, Any]]] = None,
examples: Optional[list[dict[str, Any]]] = None,
progress_bar: bool = True,
raise_on_failure: bool = True,
chat_generator: Optional[ChatGenerator] = None,
@@ -142,7 +142,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
"required to answer the following question. If no relevant sentences are found, or if you "
"believe the question cannot be answered from the given context, return an empty list, example: []"
)
self.inputs = [("questions", List[str]), ("contexts", List[List[str]])]
self.inputs = [("questions", list[str]), ("contexts", list[list[str]])]
self.outputs = ["relevant_statements"]
self.examples = examples or _DEFAULT_EXAMPLES
@@ -156,8 +156,8 @@ class ContextRelevanceEvaluator(LLMEvaluator):
progress_bar=progress_bar,
)
@component.output_types(score=float, results=List[Dict[str, Any]])
def run(self, **inputs) -> Dict[str, Any]:
@component.output_types(score=float, results=list[dict[str, Any]])
def run(self, **inputs) -> dict[str, Any]:
"""
Run the LLM evaluator.
@@ -187,7 +187,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
return result
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -203,7 +203,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ContextRelevanceEvaluator":
def from_dict(cls, data: dict[str, Any]) -> "ContextRelevanceEvaluator":
"""
Deserialize this component from a dictionary.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List
from typing import Any
from haystack import Document, component
@@ -44,10 +44,10 @@ class DocumentMAPEvaluator:
"""
# Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
@component.output_types(score=float, individual_scores=List[float])
@component.output_types(score=float, individual_scores=list[float])
def run(
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
) -> Dict[str, Any]:
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
) -> dict[str, Any]:
"""
Run the DocumentMAPEvaluator on the given inputs.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List
from typing import Any
from haystack import Document, component
@@ -42,10 +42,10 @@ class DocumentMRREvaluator:
"""
# Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
@component.output_types(score=float, individual_scores=List[float])
@component.output_types(score=float, individual_scores=list[float])
def run(
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
) -> Dict[str, Any]:
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
) -> dict[str, Any]:
"""
Run the DocumentMRREvaluator on the given inputs.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from math import log2
from typing import Any, Dict, List
from typing import Any
from haystack import Document, component
@@ -34,10 +34,10 @@ class DocumentNDCGEvaluator:
```
"""
@component.output_types(score=float, individual_scores=List[float])
@component.output_types(score=float, individual_scores=list[float])
def run(
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
) -> Dict[str, Any]:
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
) -> dict[str, Any]:
"""
Run the DocumentNDCGEvaluator on the given inputs.
@@ -68,7 +68,7 @@ class DocumentNDCGEvaluator:
return {"score": score, "individual_scores": individual_scores}
@staticmethod
def validate_inputs(gt_docs: List[List[Document]], ret_docs: List[List[Document]]):
def validate_inputs(gt_docs: list[list[Document]], ret_docs: list[list[Document]]):
"""
Validate the input parameters.
@@ -96,7 +96,7 @@ class DocumentNDCGEvaluator:
raise ValueError(msg)
@staticmethod
def calculate_dcg(gt_docs: List[Document], ret_docs: List[Document]) -> float:
def calculate_dcg(gt_docs: list[Document], ret_docs: list[Document]) -> float:
"""
Calculate the discounted cumulative gain (DCG) of the retrieved documents.
@@ -116,7 +116,7 @@ class DocumentNDCGEvaluator:
return dcg
@staticmethod
def calculate_idcg(gt_docs: List[Document]) -> float:
def calculate_idcg(gt_docs: list[Document]) -> float:
"""
Calculate the ideal discounted cumulative gain (IDCG) of the ground truth documents.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from enum import Enum
from typing import Any, Dict, List, Union
from typing import Any, Union
from haystack import component, default_to_dict, logging
from haystack.dataclasses import Document
@@ -86,7 +86,7 @@ class DocumentRecallEvaluator:
self.mode = mode
@staticmethod
def _recall_single_hit(ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> float:
def _recall_single_hit(ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
unique_truths = {g.content for g in ground_truth_documents}
unique_retrievals = {p.content for p in retrieved_documents}
retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
@@ -94,7 +94,7 @@ class DocumentRecallEvaluator:
return float(len(retrieved_ground_truths) > 0)
@staticmethod
def _recall_multi_hit(ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> float:
def _recall_multi_hit(ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
unique_truths = {g.content for g in ground_truth_documents}
unique_retrievals = {p.content for p in retrieved_documents}
retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
@@ -115,10 +115,10 @@ class DocumentRecallEvaluator:
return len(retrieved_ground_truths) / len(unique_truths)
@component.output_types(score=float, individual_scores=List[float])
@component.output_types(score=float, individual_scores=list[float])
def run(
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
) -> Dict[str, Any]:
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
) -> dict[str, Any]:
"""
Run the DocumentRecallEvaluator on the given inputs.
@@ -144,7 +144,7 @@ class DocumentRecallEvaluator:
return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from numpy import mean as np_mean
@@ -86,7 +86,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
def __init__( # pylint: disable=too-many-positional-arguments
self,
examples: Optional[List[Dict[str, Any]]] = None,
examples: Optional[list[dict[str, Any]]] = None,
progress_bar: bool = True,
raise_on_failure: bool = True,
chat_generator: Optional[ChatGenerator] = None,
@@ -132,7 +132,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
"statement made in the predicted answer. The score is 1 if the statement can be "
"inferred from the provided context or 0 if it cannot be inferred."
)
self.inputs = [("questions", List[str]), ("contexts", List[List[str]]), ("predicted_answers", List[str])]
self.inputs = [("questions", list[str]), ("contexts", list[list[str]]), ("predicted_answers", list[str])]
self.outputs = ["statements", "statement_scores"]
self.examples = examples or _DEFAULT_EXAMPLES
@@ -146,8 +146,8 @@ class FaithfulnessEvaluator(LLMEvaluator):
progress_bar=progress_bar,
)
@component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]])
def run(self, **inputs) -> Dict[str, Any]:
@component.output_types(individual_scores=list[int], score=float, results=list[dict[str, Any]])
def run(self, **inputs) -> dict[str, Any]:
"""
Run the LLM evaluator.
@@ -181,7 +181,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
return result
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -197,7 +197,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "FaithfulnessEvaluator":
def from_dict(cls, data: dict[str, Any]) -> "FaithfulnessEvaluator":
"""
Deserialize this component from a dictionary.
+13 -14
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import json
from typing import Any, Dict, List, Optional, Tuple, Type
from typing import Any, Optional
from tqdm import tqdm
@@ -30,11 +30,10 @@ class LLMEvaluator:
Usage example:
```python
from typing import List
from haystack.components.evaluators import LLMEvaluator
evaluator = LLMEvaluator(
instructions="Is this answer problematic for children?",
inputs=[("predicted_answers", List[str])],
inputs=[("predicted_answers", list[str])],
outputs=["score"],
examples=[
{"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
@@ -54,9 +53,9 @@ class LLMEvaluator:
def __init__( # pylint: disable=too-many-positional-arguments
self,
instructions: str,
inputs: List[Tuple[str, Type[List]]],
outputs: List[str],
examples: List[Dict[str, Any]],
inputs: list[tuple[str, type[list]]],
outputs: list[str],
examples: list[dict[str, Any]],
progress_bar: bool = True,
*,
raise_on_failure: bool = True,
@@ -111,7 +110,7 @@ class LLMEvaluator:
@staticmethod
def validate_init_parameters(
inputs: List[Tuple[str, Type[List]]], outputs: List[str], examples: List[Dict[str, Any]]
inputs: list[tuple[str, type[list]]], outputs: list[str], examples: list[dict[str, Any]]
):
"""
Validate the init parameters.
@@ -164,8 +163,8 @@ class LLMEvaluator:
)
raise ValueError(msg)
@component.output_types(results=List[Dict[str, Any]])
def run(self, **inputs) -> Dict[str, Any]:
@component.output_types(results=list[dict[str, Any]])
def run(self, **inputs) -> dict[str, Any]:
"""
Run the LLM evaluator.
@@ -189,7 +188,7 @@ class LLMEvaluator:
input_names, values = inputs.keys(), list(zip(*inputs.values()))
list_of_input_names_to_values = [dict(zip(input_names, v)) for v in values]
results: List[Optional[Dict[str, Any]]] = []
results: list[Optional[dict[str, Any]]] = []
metadata = []
errors = 0
for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
@@ -269,7 +268,7 @@ class LLMEvaluator:
f"Outputs:\n"
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -289,7 +288,7 @@ class LLMEvaluator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "LLMEvaluator":
def from_dict(cls, data: dict[str, Any]) -> "LLMEvaluator":
"""
Deserialize this component from a dictionary.
@@ -308,7 +307,7 @@ class LLMEvaluator:
return default_from_dict(cls, data)
@staticmethod
def validate_input_parameters(expected: Dict[str, Any], received: Dict[str, Any]) -> None:
def validate_input_parameters(expected: dict[str, Any], received: dict[str, Any]) -> None:
"""
Validate the input parameters.
@@ -345,7 +344,7 @@ class LLMEvaluator:
)
raise ValueError(msg)
def is_valid_json_and_has_expected_keys(self, expected: List[str], received: str) -> bool:
def is_valid_json_and_has_expected_keys(self, expected: list[str], received: str) -> bool:
"""
Output must be a valid JSON with the expected keys.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from numpy import mean as np_mean
@@ -82,7 +82,7 @@ class SASEvaluator:
self._token = token
self._similarity_model: Union[SentenceTransformer, CrossEncoder, None] = None
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -98,7 +98,7 @@ class SASEvaluator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SASEvaluator":
def from_dict(cls, data: dict[str, Any]) -> "SASEvaluator":
"""
Deserialize this component from a dictionary.
@@ -137,8 +137,8 @@ class SASEvaluator:
else:
self._similarity_model = SentenceTransformer(self._model, device=device, use_auth_token=token)
@component.output_types(score=float, individual_scores=List[float])
def run(self, ground_truth_answers: List[str], predicted_answers: List[str]) -> Dict[str, Any]:
@component.output_types(score=float, individual_scores=list[float])
def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
"""
SASEvaluator component run method.
@@ -4,7 +4,7 @@
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from typing import Any, Literal, Optional, Union
from jinja2 import meta
from jinja2.sandbox import SandboxedEnvironment
@@ -92,7 +92,7 @@ class LLMDocumentContentExtractor:
file_path_meta_field: str = "file_path",
root_path: Optional[str] = None,
detail: Optional[Literal["auto", "high", "low"]] = None,
size: Optional[Tuple[int, int]] = None,
size: Optional[tuple[int, int]] = None,
raise_on_failure: bool = False,
max_workers: int = 3,
):
@@ -147,7 +147,7 @@ class LLMDocumentContentExtractor:
self._chat_generator.warm_up()
self._is_warmed_up = True
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -168,7 +168,7 @@ class LLMDocumentContentExtractor:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "LLMDocumentContentExtractor":
def from_dict(cls, data: dict[str, Any]) -> "LLMDocumentContentExtractor":
"""
Deserializes the component from a dictionary.
@@ -180,7 +180,7 @@ class LLMDocumentContentExtractor:
deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
return default_from_dict(cls, data)
def _run_on_thread(self, message: Optional[ChatMessage]) -> Dict[str, Any]:
def _run_on_thread(self, message: Optional[ChatMessage]) -> dict[str, Any]:
"""
Execute the LLM inference in a separate thread for each document.
@@ -205,8 +205,8 @@ class LLMDocumentContentExtractor:
result = {"error": "LLM failed with exception: " + str(e)}
return result
@component.output_types(documents=List[Document], failed_documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
@component.output_types(documents=list[Document], failed_documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
"""
Run content extraction on a list of image-based documents using a vision-capable LLM.
@@ -225,7 +225,7 @@ class LLMDocumentContentExtractor:
# Create ChatMessage prompts for each document
image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
all_messages: List[Union[ChatMessage, None]] = []
all_messages: list[Union[ChatMessage, None]] = []
for image_content in image_contents:
if image_content is None:
# If the image content is None, it means the document could not be converted to an image.
@@ -5,7 +5,7 @@
import copy
import json
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from jinja2 import meta
from jinja2.sandbox import SandboxedEnvironment
@@ -127,8 +127,8 @@ class LLMMetadataExtractor:
self,
prompt: str,
chat_generator: ChatGenerator,
expected_keys: Optional[List[str]] = None,
page_range: Optional[List[Union[str, int]]] = None,
expected_keys: Optional[list[str]] = None,
page_range: Optional[list[Union[str, int]]] = None,
raise_on_failure: bool = False,
max_workers: int = 3,
):
@@ -172,7 +172,7 @@ class LLMMetadataExtractor:
if hasattr(self._chat_generator, "warm_up"):
self._chat_generator.warm_up()
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -191,7 +191,7 @@ class LLMMetadataExtractor:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "LLMMetadataExtractor":
def from_dict(cls, data: dict[str, Any]) -> "LLMMetadataExtractor":
"""
Deserializes the component from a dictionary.
@@ -204,8 +204,8 @@ class LLMMetadataExtractor:
deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
return default_from_dict(cls, data)
def _extract_metadata(self, llm_answer: str) -> Dict[str, Any]:
parsed_metadata: Dict[str, Any] = {}
def _extract_metadata(self, llm_answer: str) -> dict[str, Any]:
parsed_metadata: dict[str, Any] = {}
try:
parsed_metadata = json.loads(llm_answer)
@@ -229,9 +229,9 @@ class LLMMetadataExtractor:
return parsed_metadata
def _prepare_prompts(
self, documents: List[Document], expanded_range: Optional[List[int]] = None
) -> List[Union[ChatMessage, None]]:
all_prompts: List[Union[ChatMessage, None]] = []
self, documents: list[Document], expanded_range: Optional[list[int]] = None
) -> list[Union[ChatMessage, None]]:
all_prompts: list[Union[ChatMessage, None]] = []
for document in documents:
if not document.content:
logger.warning("Document {doc_id} has no content. Skipping metadata extraction.", doc_id=document.id)
@@ -257,7 +257,7 @@ class LLMMetadataExtractor:
return all_prompts
def _run_on_thread(self, prompt: Optional[ChatMessage]) -> Dict[str, Any]:
def _run_on_thread(self, prompt: Optional[ChatMessage]) -> dict[str, Any]:
# If prompt is None, return an error dictionary
if prompt is None:
return {"error": "Document has no content, skipping LLM call."}
@@ -275,8 +275,8 @@ class LLMMetadataExtractor:
result = {"error": "LLM failed with exception: " + str(e)}
return result
@component.output_types(documents=List[Document], failed_documents=List[Document])
def run(self, documents: List[Document], page_range: Optional[List[Union[str, int]]] = None):
@component.output_types(documents=list[Document], failed_documents=list[Document])
def run(self, documents: list[Document], page_range: Optional[list[Union[str, int]]] = None):
"""
Extract metadata from documents using a Large Language Model.
@@ -6,7 +6,7 @@ from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import ComponentError, DeserializationError, Document, component, default_from_dict, default_to_dict
from haystack.lazy_imports import LazyImport
@@ -110,7 +110,7 @@ class NamedEntityExtractor:
*,
backend: Union[str, NamedEntityExtractorBackend],
model: str,
pipeline_kwargs: Optional[Dict[str, Any]] = None,
pipeline_kwargs: Optional[dict[str, Any]] = None,
device: Optional[ComponentDevice] = None,
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
) -> None:
@@ -177,8 +177,8 @@ class NamedEntityExtractor:
f"Named entity extractor with backend '{self._backend.type}' failed to initialize."
) from e
@component.output_types(documents=List[Document])
def run(self, documents: List[Document], batch_size: int = 1) -> Dict[str, Any]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document], batch_size: int = 1) -> dict[str, Any]:
"""
Annotate named entities in each document and store the annotations in the document's metadata.
@@ -209,7 +209,7 @@ class NamedEntityExtractor:
return {"documents": documents}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -232,7 +232,7 @@ class NamedEntityExtractor:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "NamedEntityExtractor":
def from_dict(cls, data: dict[str, Any]) -> "NamedEntityExtractor":
"""
Deserializes the component from a dictionary.
@@ -262,7 +262,7 @@ class NamedEntityExtractor:
return self._backend.initialized
@classmethod
def get_stored_annotations(cls, document: Document) -> Optional[List[NamedEntityAnnotation]]:
def get_stored_annotations(cls, document: Document) -> Optional[list[NamedEntityAnnotation]]:
"""
Returns the document's named entity annotations stored in its metadata, if any.
@@ -284,7 +284,7 @@ class _NerBackend(ABC):
self,
_type: NamedEntityExtractorBackend,
device: ComponentDevice,
pipeline_kwargs: Optional[Dict[str, Any]] = None,
pipeline_kwargs: Optional[dict[str, Any]] = None,
) -> None:
super().__init__()
@@ -306,7 +306,7 @@ class _NerBackend(ABC):
"""
@abstractmethod
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
"""
Predict annotations for a collection of documents.
@@ -350,7 +350,7 @@ class _HfBackend(_NerBackend):
"""
def __init__(
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[Dict[str, Any]] = None
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[dict[str, Any]] = None
) -> None:
"""
Construct a Hugging Face NER backend.
@@ -382,7 +382,7 @@ class _HfBackend(_NerBackend):
self.tokenizer = AutoTokenizer.from_pretrained(self._model_name_or_path, token=token)
self.model = AutoModelForTokenClassification.from_pretrained(self._model_name_or_path, token=token)
pipeline_params: Dict[str, Any] = {
pipeline_params: dict[str, Any] = {
"task": "ner",
"model": self.model,
"tokenizer": self.tokenizer,
@@ -392,7 +392,7 @@ class _HfBackend(_NerBackend):
self.device.update_hf_kwargs(pipeline_params, overwrite=False)
self.pipeline = pipeline(**pipeline_params)
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
if not self.initialized:
raise ComponentError("Hugging Face NER backend was not initialized - Did you call `warm_up()`?")
@@ -426,7 +426,7 @@ class _SpacyBackend(_NerBackend):
"""
def __init__(
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[Dict[str, Any]] = None
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[dict[str, Any]] = None
) -> None:
"""
Construct a spaCy NER backend.
@@ -467,7 +467,7 @@ class _SpacyBackend(_NerBackend):
self._pipeline_kwargs = {k: v for k, v in self._pipeline_kwargs.items() if k not in ("texts", "batch_size")}
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
if not self.initialized:
raise ComponentError("spaCy NER backend was not initialized - Did you call `warm_up()`?")
+15 -15
View File
@@ -6,7 +6,7 @@ import asyncio
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from fnmatch import fnmatch
from typing import Callable, Dict, List, Optional, Tuple, cast
from typing import Callable, Optional, cast
import httpx
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
@@ -95,11 +95,11 @@ class LinkContentFetcher:
def __init__( # pylint: disable=too-many-positional-arguments
self,
raise_on_failure: bool = True,
user_agents: Optional[List[str]] = None,
user_agents: Optional[list[str]] = None,
retry_attempts: int = 2,
timeout: int = 3,
http2: bool = False,
client_kwargs: Optional[Dict] = None,
client_kwargs: Optional[dict] = None,
):
"""
Initializes the component.
@@ -149,7 +149,7 @@ class LinkContentFetcher:
self._async_client = httpx.AsyncClient(**client_kwargs)
# register default content handlers that extract data from the response
self.handlers: Dict[str, Callable[[httpx.Response], ByteStream]] = defaultdict(lambda: _text_content_handler)
self.handlers: dict[str, Callable[[httpx.Response], ByteStream]] = defaultdict(lambda: _text_content_handler)
self.handlers["text/*"] = _text_content_handler
self.handlers["text/html"] = _binary_content_handler
self.handlers["application/json"] = _text_content_handler
@@ -193,8 +193,8 @@ class LinkContentFetcher:
# Suppress any exceptions during cleanup
pass
@component.output_types(streams=List[ByteStream])
def run(self, urls: List[str]):
@component.output_types(streams=list[ByteStream])
def run(self, urls: list[str]):
"""
Fetches content from a list of URLs and returns a list of extracted content streams.
@@ -211,7 +211,7 @@ class LinkContentFetcher:
In all other scenarios, any retrieval errors are logged, and a list of successfully retrieved `ByteStream`
objects is returned.
"""
streams: List[ByteStream] = []
streams: list[ByteStream] = []
if not urls:
return {"streams": streams}
@@ -233,8 +233,8 @@ class LinkContentFetcher:
return {"streams": streams}
@component.output_types(streams=List[ByteStream])
async def run_async(self, urls: List[str]):
@component.output_types(streams=list[ByteStream])
async def run_async(self, urls: list[str]):
"""
Asynchronously fetches content from a list of URLs and returns a list of extracted content streams.
@@ -243,7 +243,7 @@ class LinkContentFetcher:
:param urls: A list of URLs to fetch content from.
:returns: `ByteStream` objects representing the extracted content.
"""
streams: List[ByteStream] = []
streams: list[ByteStream] = []
if not urls:
return {"streams": streams}
@@ -269,7 +269,7 @@ class LinkContentFetcher:
# At this point, result is not an exception, so we need to cast it to the correct type for mypy
if not isinstance(result, Exception): # Runtime check
# Use cast to tell mypy that result is the tuple type returned by _fetch_async
result_tuple = cast(Tuple[Optional[Dict[str, str]], Optional[ByteStream]], result)
result_tuple = cast(tuple[Optional[dict[str, str]], Optional[ByteStream]], result)
stream_metadata, stream = result_tuple
if stream_metadata is not None and stream is not None:
stream.meta.update(stream_metadata)
@@ -278,7 +278,7 @@ class LinkContentFetcher:
return {"streams": streams}
def _fetch(self, url: str) -> Tuple[Dict[str, str], ByteStream]:
def _fetch(self, url: str) -> tuple[dict[str, str], ByteStream]:
"""
Fetches content from a URL and returns it as a ByteStream.
@@ -312,7 +312,7 @@ class LinkContentFetcher:
async def _fetch_async(
self, url: str, client: httpx.AsyncClient
) -> Tuple[Optional[Dict[str, str]], Optional[ByteStream]]:
) -> tuple[Optional[dict[str, str]], Optional[ByteStream]]:
"""
Asynchronously fetches content from a URL and returns it as a ByteStream.
@@ -322,7 +322,7 @@ class LinkContentFetcher:
"""
content_type: str = "text/html"
stream: Optional[ByteStream] = None
metadata: Optional[Dict[str, str]] = None
metadata: Optional[dict[str, str]] = None
try:
response = await self._get_response_async(url, client)
@@ -342,7 +342,7 @@ class LinkContentFetcher:
return metadata, stream
def _fetch_with_exception_suppression(self, url: str) -> Tuple[Optional[Dict[str, str]], Optional[ByteStream]]:
def _fetch_with_exception_suppression(self, url: str) -> tuple[Optional[dict[str, str]], Optional[ByteStream]]:
"""
Fetches content from a URL and returns it as a ByteStream.
+6 -6
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, Optional
from typing import Any, Optional
from openai.lib.azure import AzureADTokenProvider, AzureOpenAI
@@ -67,9 +67,9 @@ class AzureOpenAIGenerator(OpenAIGenerator):
system_prompt: Optional[str] = None,
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
default_headers: Optional[Dict[str, str]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
default_headers: Optional[dict[str, str]] = None,
*,
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
):
@@ -164,7 +164,7 @@ class AzureOpenAIGenerator(OpenAIGenerator):
default_headers=self.default_headers,
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -194,7 +194,7 @@ class AzureOpenAIGenerator(OpenAIGenerator):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIGenerator":
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIGenerator":
"""
Deserialize this component from a dictionary.
+8 -8
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from openai.lib.azure import AsyncAzureADTokenProvider, AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
@@ -80,13 +80,13 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
streaming_callback: Optional[StreamingCallbackT] = None,
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
default_headers: Optional[Dict[str, str]] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
default_headers: Optional[dict[str, str]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tools_strict: bool = False,
*,
azure_ad_token_provider: Optional[Union[AzureADTokenProvider, AsyncAzureADTokenProvider]] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Initialize the Azure OpenAI Chat Generator component.
@@ -170,7 +170,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
self.tools = tools
self.tools_strict = tools_strict
client_args: Dict[str, Any] = {
client_args: dict[str, Any] = {
"api_version": api_version,
"azure_endpoint": azure_endpoint,
"azure_deployment": azure_deployment,
@@ -190,7 +190,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_args
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -221,7 +221,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIChatGenerator":
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIChatGenerator":
"""
Deserialize this component from a dictionary.
@@ -4,7 +4,7 @@
import json
from datetime import datetime
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Union
from typing import Any, AsyncIterable, Iterable, Optional, Union
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message
@@ -47,7 +47,7 @@ with LazyImport(message="Run 'pip install \"huggingface_hub[inference]>=0.27.0\"
)
def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[List["ChatCompletionOutputToolCall"]]) -> List[ToolCall]:
def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[list["ChatCompletionOutputToolCall"]]) -> list[ToolCall]:
"""
Convert HuggingFace API tool calls to a list of Haystack ToolCall.
@@ -93,8 +93,8 @@ def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[List["ChatCompletionOut
def _convert_tools_to_hfapi_tools(
tools: Optional[Union[List[Tool], Toolset]],
) -> Optional[List["ChatCompletionInputTool"]]:
tools: Optional[Union[list[Tool], Toolset]],
) -> Optional[list["ChatCompletionInputTool"]]:
if not tools:
return None
@@ -140,7 +140,7 @@ def _map_hf_finish_reason_to_haystack(choice: "ChatCompletionStreamOutputChoice"
return "tool_calls"
# Map HuggingFace finish reasons to Haystack standard ones
mapping: Dict[str, FinishReason] = {
mapping: dict[str, FinishReason] = {
"length": "length", # Direct match
"eos_token": "stop", # EOS token means natural stop
"stop_sequence": "stop", # Stop sequence means natural stop
@@ -151,7 +151,7 @@ def _map_hf_finish_reason_to_haystack(choice: "ChatCompletionStreamOutputChoice"
def _convert_chat_completion_stream_output_to_streaming_chunk(
chunk: "ChatCompletionStreamOutput",
previous_chunks: List[StreamingChunk],
previous_chunks: list[StreamingChunk],
component_info: Optional[ComponentInfo] = None,
) -> StreamingChunk:
"""
@@ -287,12 +287,12 @@ class HuggingFaceAPIChatGenerator:
def __init__( # pylint: disable=too-many-positional-arguments
self,
api_type: Union[HFGenerationAPIType, str],
api_params: Dict[str, str],
api_params: dict[str, str],
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
generation_kwargs: Optional[Dict[str, Any]] = None,
stop_words: Optional[List[str]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
stop_words: Optional[list[str]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
):
"""
Initialize the HuggingFaceAPIChatGenerator instance.
@@ -372,7 +372,7 @@ class HuggingFaceAPIChatGenerator:
self.generation_kwargs = generation_kwargs
self.streaming_callback = streaming_callback
resolved_api_params: Dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
resolved_api_params: dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
self._client = InferenceClient(
model_or_url, token=token.resolve_value() if token else None, **resolved_api_params
)
@@ -381,7 +381,7 @@ class HuggingFaceAPIChatGenerator:
)
self.tools = tools
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -400,7 +400,7 @@ class HuggingFaceAPIChatGenerator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIChatGenerator":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIChatGenerator":
"""
Deserialize this component from a dictionary.
"""
@@ -412,12 +412,12 @@ class HuggingFaceAPIChatGenerator:
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
return default_from_dict(cls, data)
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
def run(
self,
messages: List[ChatMessage],
generation_kwargs: Optional[Dict[str, Any]] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
messages: list[ChatMessage],
generation_kwargs: Optional[dict[str, Any]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
):
"""
@@ -463,12 +463,12 @@ class HuggingFaceAPIChatGenerator:
return self._run_non_streaming(formatted_messages, generation_kwargs, hf_tools)
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
async def run_async(
self,
messages: List[ChatMessage],
generation_kwargs: Optional[Dict[str, Any]] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
messages: list[ChatMessage],
generation_kwargs: Optional[dict[str, Any]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
):
"""
@@ -517,8 +517,8 @@ class HuggingFaceAPIChatGenerator:
def _run_streaming(
self,
messages: List[Dict[str, str]],
generation_kwargs: Dict[str, Any],
messages: list[dict[str, str]],
generation_kwargs: dict[str, Any],
streaming_callback: SyncStreamingCallbackT,
):
api_output: Iterable[ChatCompletionStreamOutput] = self._client.chat_completion(
@@ -529,7 +529,7 @@ class HuggingFaceAPIChatGenerator:
)
component_info = ComponentInfo.from_component(self)
streaming_chunks: List[StreamingChunk] = []
streaming_chunks: list[StreamingChunk] = []
for chunk in api_output:
streaming_chunk = _convert_chat_completion_stream_output_to_streaming_chunk(
chunk=chunk, previous_chunks=streaming_chunks, component_info=component_info
@@ -545,10 +545,10 @@ class HuggingFaceAPIChatGenerator:
def _run_non_streaming(
self,
messages: List[Dict[str, str]],
generation_kwargs: Dict[str, Any],
tools: Optional[List["ChatCompletionInputTool"]] = None,
) -> Dict[str, List[ChatMessage]]:
messages: list[dict[str, str]],
generation_kwargs: dict[str, Any],
tools: Optional[list["ChatCompletionInputTool"]] = None,
) -> dict[str, list[ChatMessage]]:
api_chat_output: ChatCompletionOutput = self._client.chat_completion(
messages=messages, tools=tools, **generation_kwargs
)
@@ -565,7 +565,7 @@ class HuggingFaceAPIChatGenerator:
tool_calls = _convert_hfapi_tool_calls(choice.message.tool_calls)
meta: Dict[str, Any] = {
meta: dict[str, Any] = {
"model": self._client.model,
"finish_reason": choice.finish_reason,
"index": choice.index,
@@ -584,8 +584,8 @@ class HuggingFaceAPIChatGenerator:
async def _run_streaming_async(
self,
messages: List[Dict[str, str]],
generation_kwargs: Dict[str, Any],
messages: list[dict[str, str]],
generation_kwargs: dict[str, Any],
streaming_callback: AsyncStreamingCallbackT,
):
api_output: AsyncIterable[ChatCompletionStreamOutput] = await self._async_client.chat_completion(
@@ -596,7 +596,7 @@ class HuggingFaceAPIChatGenerator:
)
component_info = ComponentInfo.from_component(self)
streaming_chunks: List[StreamingChunk] = []
streaming_chunks: list[StreamingChunk] = []
async for chunk in api_output:
stream_chunk = _convert_chat_completion_stream_output_to_streaming_chunk(
chunk=chunk, previous_chunks=streaming_chunks, component_info=component_info
@@ -612,10 +612,10 @@ class HuggingFaceAPIChatGenerator:
async def _run_non_streaming_async(
self,
messages: List[Dict[str, str]],
generation_kwargs: Dict[str, Any],
tools: Optional[List["ChatCompletionInputTool"]] = None,
) -> Dict[str, List[ChatMessage]]:
messages: list[dict[str, str]],
generation_kwargs: dict[str, Any],
tools: Optional[list["ChatCompletionInputTool"]] = None,
) -> dict[str, list[ChatMessage]]:
api_chat_output: ChatCompletionOutput = await self._async_client.chat_completion(
messages=messages, tools=tools, **generation_kwargs
)
@@ -629,7 +629,7 @@ class HuggingFaceAPIChatGenerator:
tool_calls = _convert_hfapi_tool_calls(choice.message.tool_calls)
meta: Dict[str, Any] = {
meta: dict[str, Any] = {
"model": self._async_client.model,
"finish_reason": choice.finish_reason,
"index": choice.index,
@@ -8,7 +8,7 @@ import re
import sys
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, suppress
from typing import Any, Callable, Dict, List, Literal, Optional, Union, cast
from typing import Any, Callable, Literal, Optional, Union, cast
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.dataclasses import AsyncStreamingCallbackT, ChatMessage, ComponentInfo, StreamingCallbackT, ToolCall
@@ -57,7 +57,7 @@ DEFAULT_TOOL_PATTERN = (
)
def default_tool_parser(text: str) -> Optional[List[ToolCall]]:
def default_tool_parser(text: str) -> Optional[list[ToolCall]]:
"""
Default implementation for parsing tool calls from model output text.
@@ -131,12 +131,12 @@ class HuggingFaceLocalChatGenerator:
device: Optional[ComponentDevice] = None,
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
chat_template: Optional[str] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
stop_words: Optional[List[str]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
stop_words: Optional[list[str]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
tool_parsing_function: Optional[Callable[[str], Optional[List[ToolCall]]]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tool_parsing_function: Optional[Callable[[str], Optional[list[ToolCall]]]] = None,
async_executor: Optional[ThreadPoolExecutor] = None,
) -> None:
"""
@@ -263,7 +263,7 @@ class HuggingFaceLocalChatGenerator:
if self._owns_executor:
self.executor.shutdown(wait=True)
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
@@ -278,7 +278,7 @@ class HuggingFaceLocalChatGenerator:
if self.pipeline is None:
self.pipeline = pipeline(**self.huggingface_pipeline_kwargs)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -304,7 +304,7 @@ class HuggingFaceLocalChatGenerator:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceLocalChatGenerator":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceLocalChatGenerator":
"""
Deserializes the component from a dictionary.
@@ -329,14 +329,14 @@ class HuggingFaceLocalChatGenerator:
deserialize_hf_model_kwargs(huggingface_pipeline_kwargs)
return default_from_dict(cls, data)
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
def run(
self,
messages: List[ChatMessage],
generation_kwargs: Optional[Dict[str, Any]] = None,
messages: list[ChatMessage],
generation_kwargs: Optional[dict[str, Any]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
) -> Dict[str, List[ChatMessage]]:
tools: Optional[Union[list[Tool], Toolset]] = None,
) -> dict[str, list[ChatMessage]]:
"""
Invoke text generation inference based on the provided messages and generation parameters.
@@ -380,7 +380,7 @@ class HuggingFaceLocalChatGenerator:
index: int,
tokenizer: Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"],
prompt: str,
generation_kwargs: Dict[str, Any],
generation_kwargs: dict[str, Any],
parse_tool_calls: bool = False,
) -> ChatMessage:
"""
@@ -424,7 +424,7 @@ class HuggingFaceLocalChatGenerator:
return ChatMessage.from_assistant(tool_calls=tool_calls, text=None if tool_calls else text, meta=meta)
@staticmethod
def _validate_stop_words(stop_words: Optional[List[str]]) -> Optional[List[str]]:
def _validate_stop_words(stop_words: Optional[list[str]]) -> Optional[list[str]]:
"""
Validates the provided stop words.
@@ -441,14 +441,14 @@ class HuggingFaceLocalChatGenerator:
return list(set(stop_words or []))
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
async def run_async(
self,
messages: List[ChatMessage],
generation_kwargs: Optional[Dict[str, Any]] = None,
messages: list[ChatMessage],
generation_kwargs: Optional[dict[str, Any]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
) -> Dict[str, List[ChatMessage]]:
tools: Optional[Union[list[Tool], Toolset]] = None,
) -> dict[str, list[ChatMessage]]:
"""
Asynchronously invokes text generation inference based on the provided messages and generation parameters.
@@ -518,11 +518,11 @@ class HuggingFaceLocalChatGenerator:
def _prepare_inputs(
self,
messages: List[ChatMessage],
generation_kwargs: Optional[Dict[str, Any]] = None,
messages: list[ChatMessage],
generation_kwargs: Optional[dict[str, Any]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
) -> Dict[str, Any]:
tools: Optional[Union[list[Tool], Toolset]] = None,
) -> dict[str, Any]:
"""
Prepares the inputs for the Hugging Face pipeline.
@@ -601,13 +601,13 @@ class HuggingFaceLocalChatGenerator:
def _convert_hf_output_to_chat_messages(
self,
*,
hf_pipeline_output: List[Dict[str, Any]],
hf_pipeline_output: list[dict[str, Any]],
prepared_prompt: str,
tokenizer: Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"],
generation_kwargs: Dict[str, Any],
stop_words: Optional[List[str]],
tools: Optional[Union[List[Tool], Toolset]] = None,
) -> List[ChatMessage]:
generation_kwargs: dict[str, Any],
stop_words: Optional[list[str]],
tools: Optional[Union[list[Tool], Toolset]] = None,
) -> list[ChatMessage]:
"""
Converts the HuggingFace pipeline output into a List of ChatMessages
+27 -27
View File
@@ -5,7 +5,7 @@
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
@@ -90,12 +90,12 @@ class OpenAIChatGenerator:
streaming_callback: Optional[StreamingCallbackT] = None,
api_base_url: Optional[str] = None,
organization: Optional[str] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tools_strict: bool = False,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Creates an instance of OpenAIChatGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-4o-mini
@@ -168,7 +168,7 @@ class OpenAIChatGenerator:
if max_retries is None:
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
client_kwargs: Dict[str, Any] = {
client_kwargs: dict[str, Any] = {
"api_key": api_key.resolve_value(),
"organization": organization,
"base_url": api_base_url,
@@ -181,13 +181,13 @@ class OpenAIChatGenerator:
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
)
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -211,7 +211,7 @@ class OpenAIChatGenerator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIChatGenerator":
def from_dict(cls, data: dict[str, Any]) -> "OpenAIChatGenerator":
"""
Deserialize this component from a dictionary.
@@ -227,14 +227,14 @@ class OpenAIChatGenerator:
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
return default_from_dict(cls, data)
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
def run(
self,
messages: List[ChatMessage],
messages: list[ChatMessage],
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
*,
tools: Optional[Union[List[Tool], Toolset]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tools_strict: Optional[bool] = None,
):
"""
@@ -299,14 +299,14 @@ class OpenAIChatGenerator:
return {"replies": completions}
@component.output_types(replies=List[ChatMessage])
@component.output_types(replies=list[ChatMessage])
async def run_async(
self,
messages: List[ChatMessage],
messages: list[ChatMessage],
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
*,
tools: Optional[Union[List[Tool], Toolset]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tools_strict: Optional[bool] = None,
):
"""
@@ -380,12 +380,12 @@ class OpenAIChatGenerator:
def _prepare_api_call( # noqa: PLR0913
self,
*,
messages: List[ChatMessage],
messages: list[ChatMessage],
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
tools: Optional[Union[List[Tool], Toolset]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
tools: Optional[Union[list[Tool], Toolset]] = None,
tools_strict: Optional[bool] = None,
) -> Dict[str, Any]:
) -> dict[str, Any]:
# update generation kwargs by merging with the generation kwargs passed to the run method
generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
@@ -423,9 +423,9 @@ class OpenAIChatGenerator:
**generation_kwargs,
}
def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> List[ChatMessage]:
def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> list[ChatMessage]:
component_info = ComponentInfo.from_component(self)
chunks: List[StreamingChunk] = []
chunks: list[StreamingChunk] = []
for chunk in chat_completion: # pylint: disable=not-an-iterable
assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
@@ -437,9 +437,9 @@ class OpenAIChatGenerator:
async def _handle_async_stream_response(
self, chat_completion: AsyncStream, callback: AsyncStreamingCallbackT
) -> List[ChatMessage]:
) -> list[ChatMessage]:
component_info = ComponentInfo.from_component(self)
chunks: List[StreamingChunk] = []
chunks: list[StreamingChunk] = []
async for chunk in chat_completion: # pylint: disable=not-an-iterable
assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
@@ -450,7 +450,7 @@ class OpenAIChatGenerator:
return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
def _check_finish_reason(meta: Dict[str, Any]) -> None:
def _check_finish_reason(meta: dict[str, Any]) -> None:
if meta["finish_reason"] == "length":
logger.warning(
"The completion for index {index} has been truncated before reaching a natural stopping point. "
@@ -507,7 +507,7 @@ def _convert_chat_completion_to_chat_message(completion: ChatCompletion, choice:
def _convert_chat_completion_chunk_to_streaming_chunk(
chunk: ChatCompletionChunk, previous_chunks: List[StreamingChunk], component_info: Optional[ComponentInfo] = None
chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: Optional[ComponentInfo] = None
) -> StreamingChunk:
"""
Converts the streaming response chunk from the OpenAI API to a StreamingChunk.
@@ -520,7 +520,7 @@ def _convert_chat_completion_chunk_to_streaming_chunk(
:returns:
A StreamingChunk object representing the content of the chunk from the OpenAI API.
"""
finish_reason_mapping: Dict[str, FinishReason] = {
finish_reason_mapping: dict[str, FinishReason] = {
"stop": "stop",
"length": "length",
"content_filter": "content_filter",
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Protocol, TypeVar
from typing import Any, Protocol, TypeVar
from haystack.dataclasses import ChatMessage
@@ -21,12 +21,12 @@ class ChatGenerator(Protocol):
responses using a Language Model. They return a dictionary.
"""
def run(self, messages: List[ChatMessage]) -> Dict[str, Any]:
def run(self, messages: list[ChatMessage]) -> dict[str, Any]:
"""
Generate messages using the underlying Language Model.
Implementing classes may accept additional optional parameters in their run method.
For example: `def run (self, messages: List[ChatMessage], param_a="default", param_b="another_default")`.
For example: `def run (self, messages: list[ChatMessage], param_a="default", param_b="another_default")`.
:param messages:
A list of ChatMessage instances representing the input messages.
@@ -4,7 +4,7 @@
from dataclasses import asdict
from datetime import datetime
from typing import Any, Dict, Iterable, List, Optional, Union, cast
from typing import Any, Iterable, Optional, Union, cast
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.dataclasses import (
@@ -95,10 +95,10 @@ class HuggingFaceAPIGenerator:
def __init__( # pylint: disable=too-many-positional-arguments
self,
api_type: Union[HFGenerationAPIType, str],
api_params: Dict[str, str],
api_params: dict[str, str],
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
generation_kwargs: Optional[Dict[str, Any]] = None,
stop_words: Optional[List[str]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
stop_words: Optional[list[str]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
):
"""
@@ -173,12 +173,12 @@ class HuggingFaceAPIGenerator:
self.generation_kwargs = generation_kwargs
self.streaming_callback = streaming_callback
resolved_api_params: Dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
resolved_api_params: dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
self._client = InferenceClient(
model_or_url, token=token.resolve_value() if token else None, **resolved_api_params
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -196,7 +196,7 @@ class HuggingFaceAPIGenerator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIGenerator":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIGenerator":
"""
Deserialize this component from a dictionary.
"""
@@ -207,12 +207,12 @@ class HuggingFaceAPIGenerator:
init_params["streaming_callback"] = deserialize_callable(serialized_callback_handler)
return default_from_dict(cls, data)
@component.output_types(replies=List[str], meta=List[Dict[str, Any]])
@component.output_types(replies=list[str], meta=list[dict[str, Any]])
def run(
self,
prompt: str,
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
):
"""
Invoke the text generation inference for the given prompt and generation parameters.
@@ -251,7 +251,7 @@ class HuggingFaceAPIGenerator:
def _stream_and_build_response(
self, hf_output: Iterable["TextGenerationStreamOutput"], streaming_callback: SyncStreamingCallbackT
):
chunks: List[StreamingChunk] = []
chunks: list[StreamingChunk] = []
first_chunk_time = None
component_info = ComponentInfo.from_component(self)
@@ -264,7 +264,7 @@ class HuggingFaceAPIGenerator:
if first_chunk_time is None:
first_chunk_time = datetime.now().isoformat()
mapping: Dict[str, FinishReason] = {
mapping: dict[str, FinishReason] = {
"length": "length", # Direct match
"eos_token": "stop", # EOS token means natural stop
"stop_sequence": "stop", # Stop sequence means natural stop
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Optional, cast
from typing import Any, Literal, Optional, cast
from haystack import component, default_from_dict, default_to_dict, logging
from haystack.dataclasses import ComponentInfo, StreamingCallbackT, select_streaming_callback
@@ -61,9 +61,9 @@ class HuggingFaceLocalGenerator:
task: Optional[Literal["text-generation", "text2text-generation"]] = None,
device: Optional[ComponentDevice] = None,
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
generation_kwargs: Optional[Dict[str, Any]] = None,
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
stop_words: Optional[List[str]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
stop_words: Optional[list[str]] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
):
"""
@@ -131,7 +131,7 @@ class HuggingFaceLocalGenerator:
self.stopping_criteria_list: Optional[StoppingCriteriaList] = None
self.streaming_callback = streaming_callback
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
@@ -164,7 +164,7 @@ class HuggingFaceLocalGenerator:
)
self.stopping_criteria_list = StoppingCriteriaList([stop_words_criteria])
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -188,7 +188,7 @@ class HuggingFaceLocalGenerator:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceLocalGenerator":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceLocalGenerator":
"""
Deserializes the component from a dictionary.
@@ -207,12 +207,12 @@ class HuggingFaceLocalGenerator:
deserialize_hf_model_kwargs(huggingface_pipeline_kwargs)
return default_from_dict(cls, data)
@component.output_types(replies=List[str])
@component.output_types(replies=list[str])
def run(
self,
prompt: str,
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
):
"""
Run the text generation model on the given prompt.
+10 -10
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from openai import OpenAI, Stream
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@@ -69,10 +69,10 @@ class OpenAIGenerator:
api_base_url: Optional[str] = None,
organization: Optional[str] = None,
system_prompt: Optional[str] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Creates an instance of OpenAIGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-4o-mini
@@ -141,13 +141,13 @@ class OpenAIGenerator:
http_client=init_http_client(self.http_client_kwargs, async_client=False),
)
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -168,7 +168,7 @@ class OpenAIGenerator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIGenerator":
def from_dict(cls, data: dict[str, Any]) -> "OpenAIGenerator":
"""
Deserialize this component from a dictionary.
@@ -184,13 +184,13 @@ class OpenAIGenerator:
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
return default_from_dict(cls, data)
@component.output_types(replies=List[str], meta=List[Dict[str, Any]])
@component.output_types(replies=list[str], meta=list[dict[str, Any]])
def run(
self,
prompt: str,
system_prompt: Optional[str] = None,
streaming_callback: Optional[StreamingCallbackT] = None,
generation_kwargs: Optional[Dict[str, Any]] = None,
generation_kwargs: Optional[dict[str, Any]] = None,
):
"""
Invoke the text generation inference based on the provided messages and generation parameters.
@@ -236,14 +236,14 @@ class OpenAIGenerator:
**generation_kwargs,
)
completions: List[ChatMessage] = []
completions: list[ChatMessage] = []
if streaming_callback is not None:
num_responses = generation_kwargs.pop("n", 1)
if num_responses > 1:
raise ValueError("Cannot stream multiple responses, please set n=1.")
component_info = ComponentInfo.from_component(self)
chunks: List[StreamingChunk] = []
chunks: list[StreamingChunk] = []
for chunk in completion:
chunk_delta: StreamingChunk = _convert_chat_completion_chunk_to_streaming_chunk(
chunk=chunk, # type: ignore
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import os
from typing import Any, Dict, List, Literal, Optional
from typing import Any, Literal, Optional
from openai import OpenAI
from openai.types.image import Image
@@ -42,7 +42,7 @@ class DALLEImageGenerator:
organization: Optional[str] = None,
timeout: Optional[float] = None,
max_retries: Optional[int] = None,
http_client_kwargs: Optional[Dict[str, Any]] = None,
http_client_kwargs: Optional[dict[str, Any]] = None,
):
"""
Creates an instance of DALLEImageGenerator. Unless specified otherwise in `model`, uses OpenAI's dall-e-3.
@@ -94,7 +94,7 @@ class DALLEImageGenerator:
http_client=init_http_client(self.http_client_kwargs, async_client=False),
)
@component.output_types(images=List[str], revised_prompt=str)
@component.output_types(images=list[str], revised_prompt=str)
def run(
self,
prompt: str,
@@ -136,7 +136,7 @@ class DALLEImageGenerator:
revised_prompt = ""
return {"images": [image_str], "revised_prompt": revised_prompt}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize this component to a dictionary.
@@ -156,7 +156,7 @@ class DALLEImageGenerator:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DALLEImageGenerator":
def from_dict(cls, data: dict[str, Any]) -> "DALLEImageGenerator":
"""
Deserialize this component from a dictionary.
+2 -3
View File
@@ -3,7 +3,6 @@
# SPDX-License-Identifier: Apache-2.0
import json
from typing import Dict, List
from haystack import logging
from haystack.dataclasses import ChatMessage, StreamingChunk, ToolCall
@@ -69,7 +68,7 @@ def print_streaming_chunk(chunk: StreamingChunk) -> None:
print("\n\n", flush=True, end="")
def _convert_streaming_chunks_to_chat_message(chunks: List[StreamingChunk]) -> ChatMessage:
def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> ChatMessage:
"""
Connects the streaming chunks into a single ChatMessage.
@@ -81,7 +80,7 @@ def _convert_streaming_chunks_to_chat_message(chunks: List[StreamingChunk]) -> C
tool_calls = []
# Process tool calls if present in any chunk
tool_call_data: Dict[int, Dict[str, str]] = {} # Track tool calls by index
tool_call_data: dict[int, dict[str, str]] = {} # Track tool calls by index
for chunk in chunks:
if chunk.tool_calls:
for tool_call in chunk.tool_calls:
+9 -9
View File
@@ -5,7 +5,7 @@
import itertools
from enum import Enum
from math import inf
from typing import Any, Callable, Dict, List, Optional, Union
from typing import Any, Callable, Optional, Union
from haystack import component, default_from_dict, default_to_dict
from haystack.core.component.types import Variadic
@@ -103,16 +103,16 @@ class AnswerJoiner:
"""
if isinstance(join_mode, str):
join_mode = JoinMode.from_str(join_mode)
join_mode_functions: Dict[JoinMode, Callable[[List[List[AnswerType]]], List[AnswerType]]] = {
join_mode_functions: dict[JoinMode, Callable[[list[list[AnswerType]]], list[AnswerType]]] = {
JoinMode.CONCATENATE: self._concatenate
}
self.join_mode_function: Callable[[List[List[AnswerType]]], List[AnswerType]] = join_mode_functions[join_mode]
self.join_mode_function: Callable[[list[list[AnswerType]]], list[AnswerType]] = join_mode_functions[join_mode]
self.join_mode = join_mode
self.top_k = top_k
self.sort_by_score = sort_by_score
@component.output_types(answers=List[AnswerType])
def run(self, answers: Variadic[List[AnswerType]], top_k: Optional[int] = None):
@component.output_types(answers=list[AnswerType])
def run(self, answers: Variadic[list[AnswerType]], top_k: Optional[int] = None):
"""
Joins multiple lists of Answers into a single list depending on the `join_mode` parameter.
@@ -128,7 +128,7 @@ class AnswerJoiner:
"""
answers_list = list(answers)
join_function = self.join_mode_function
output_answers: List[AnswerType] = join_function(answers_list)
output_answers: list[AnswerType] = join_function(answers_list)
if self.sort_by_score:
output_answers = sorted(
@@ -140,7 +140,7 @@ class AnswerJoiner:
output_answers = output_answers[:top_k]
return {"answers": output_answers}
def _concatenate(self, answer_lists: List[List[AnswerType]]) -> List[AnswerType]:
def _concatenate(self, answer_lists: list[list[AnswerType]]) -> list[AnswerType]:
"""
Concatenate multiple lists of Answers, flattening them into a single list and sorting by score.
@@ -148,7 +148,7 @@ class AnswerJoiner:
"""
return list(itertools.chain.from_iterable(answer_lists))
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -158,7 +158,7 @@ class AnswerJoiner:
return default_to_dict(self, join_mode=str(self.join_mode), top_k=self.top_k, sort_by_score=self.sort_by_score)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AnswerJoiner":
def from_dict(cls, data: dict[str, Any]) -> "AnswerJoiner":
"""
Deserializes the component from a dictionary.
+9 -10
View File
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, Type
from typing import Any
from haystack import component, default_from_dict, default_to_dict
from haystack.core.component.types import GreedyVariadic
@@ -31,7 +31,6 @@ class BranchJoiner:
### Example Usage:
```python
import json
from typing import List
from haystack import Pipeline
from haystack.components.converters import OutputAdapter
@@ -55,10 +54,10 @@ class BranchJoiner:
pipe = Pipeline()
# Add components to the pipeline
pipe.add_component('joiner', BranchJoiner(List[ChatMessage]))
pipe.add_component('joiner', BranchJoiner(list[ChatMessage]))
pipe.add_component('generator', OpenAIChatGenerator(model="gpt-4o-mini"))
pipe.add_component('validator', JsonSchemaValidator(json_schema=person_schema))
pipe.add_component('adapter', OutputAdapter("{{chat_message}}", List[ChatMessage], unsafe=True))
pipe.add_component('adapter', OutputAdapter("{{chat_message}}", list[ChatMessage], unsafe=True))
# And connect them
pipe.connect("adapter", "joiner")
@@ -80,16 +79,16 @@ class BranchJoiner:
```
Note that `BranchJoiner` can manage only one data type at a time. In this case, `BranchJoiner` is created for
passing `List[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
passing `list[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
connected components and also the type of data that `BranchJoiner` will send through its output.
In the code example, `BranchJoiner` receives a looped back `List[ChatMessage]` from the `JsonSchemaValidator` and
In the code example, `BranchJoiner` receives a looped back `list[ChatMessage]` from the `JsonSchemaValidator` and
sends it down to the `OpenAIChatGenerator` for re-generation. We can have multiple loopback connections in the
pipeline. In this instance, the downstream component is only one (the `OpenAIChatGenerator`), but the pipeline could
have more than one downstream component.
"""
def __init__(self, type_: Type):
def __init__(self, type_: type):
"""
Creates a `BranchJoiner` component.
@@ -99,7 +98,7 @@ class BranchJoiner:
component.set_input_types(self, value=GreedyVariadic[type_]) # type: ignore
component.set_output_types(self, value=type_)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component into a dictionary.
@@ -109,7 +108,7 @@ class BranchJoiner:
return default_to_dict(self, type_=serialize_type(self.type_))
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "BranchJoiner":
def from_dict(cls, data: dict[str, Any]) -> "BranchJoiner":
"""
Deserializes a `BranchJoiner` instance from a dictionary.
@@ -120,7 +119,7 @@ class BranchJoiner:
data["init_parameters"]["type_"] = deserialize_type(data["init_parameters"]["type_"])
return default_from_dict(cls, data)
def run(self, **kwargs) -> Dict[str, Any]:
def run(self, **kwargs) -> dict[str, Any]:
"""
Executes the `BranchJoiner`, selecting the first available input value and passing it downstream.
+10 -10
View File
@@ -6,7 +6,7 @@ import itertools
from collections import defaultdict
from enum import Enum
from math import inf
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict, logging
from haystack.core.component.types import Variadic
@@ -87,7 +87,7 @@ class DocumentJoiner:
def __init__(
self,
join_mode: Union[str, JoinMode] = JoinMode.CONCATENATE,
weights: Optional[List[float]] = None,
weights: Optional[list[float]] = None,
top_k: Optional[int] = None,
sort_by_score: bool = True,
):
@@ -126,8 +126,8 @@ class DocumentJoiner:
self.top_k = top_k
self.sort_by_score = sort_by_score
@component.output_types(documents=List[Document])
def run(self, documents: Variadic[List[Document]], top_k: Optional[int] = None):
@component.output_types(documents=list[Document])
def run(self, documents: Variadic[list[Document]], top_k: Optional[int] = None):
"""
Joins multiple lists of Documents into a single list depending on the `join_mode` parameter.
@@ -161,7 +161,7 @@ class DocumentJoiner:
return {"documents": output_documents}
@staticmethod
def _concatenate(document_lists: List[List[Document]]) -> List[Document]:
def _concatenate(document_lists: list[list[Document]]) -> list[Document]:
"""
Concatenate multiple lists of Documents and return only the Document with the highest score for duplicates.
"""
@@ -174,7 +174,7 @@ class DocumentJoiner:
output.append(doc_with_best_score)
return output
def _merge(self, document_lists: List[List[Document]]) -> List[Document]:
def _merge(self, document_lists: list[list[Document]]) -> list[Document]:
"""
Merge multiple lists of Documents and calculate a weighted sum of the scores of duplicate Documents.
"""
@@ -196,7 +196,7 @@ class DocumentJoiner:
return list(documents_map.values())
def _reciprocal_rank_fusion(self, document_lists: List[List[Document]]) -> List[Document]:
def _reciprocal_rank_fusion(self, document_lists: list[list[Document]]) -> list[Document]:
"""
Merge multiple lists of Documents and assign scores based on reciprocal rank fusion.
@@ -230,7 +230,7 @@ class DocumentJoiner:
return list(documents_map.values())
@staticmethod
def _distribution_based_rank_fusion(document_lists: List[List[Document]]) -> List[Document]:
def _distribution_based_rank_fusion(document_lists: list[list[Document]]) -> list[Document]:
"""
Merge multiple lists of Documents and assign scores based on Distribution-Based Score Fusion.
@@ -260,7 +260,7 @@ class DocumentJoiner:
return output
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -276,7 +276,7 @@ class DocumentJoiner:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DocumentJoiner":
def from_dict(cls, data: dict[str, Any]) -> "DocumentJoiner":
"""
Deserializes the component from a dictionary.
+8 -9
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from itertools import chain
from typing import Any, Dict, List, Optional, Type
from typing import Any, Optional
from haystack import component, default_from_dict, default_to_dict
from haystack.core.component.types import Variadic
@@ -25,7 +25,6 @@ class ListJoiner:
from haystack.dataclasses import ChatMessage
from haystack import Pipeline
from haystack.components.joiners import ListJoiner
from typing import List
user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")]
@@ -48,7 +47,7 @@ class ListJoiner:
pipe.add_component("llm", llm)
pipe.add_component("feedback_prompt_builder", feedback_prompt_builder)
pipe.add_component("feedback_llm", feedback_llm)
pipe.add_component("list_joiner", ListJoiner(List[ChatMessage]))
pipe.add_component("list_joiner", ListJoiner(list[ChatMessage]))
pipe.connect("prompt_builder.prompt", "llm.messages")
pipe.connect("prompt_builder.prompt", "list_joiner")
@@ -65,11 +64,11 @@ class ListJoiner:
```
"""
def __init__(self, list_type_: Optional[Type] = None):
def __init__(self, list_type_: Optional[type] = None):
"""
Creates a ListJoiner component.
:param list_type_: The expected type of the lists this component will join (e.g., List[ChatMessage]).
:param list_type_: The expected type of the lists this component will join (e.g., list[ChatMessage]).
If specified, all input lists must conform to this type. If None, the component defaults to handling
lists of any type including mixed types.
"""
@@ -77,9 +76,9 @@ class ListJoiner:
if list_type_ is not None:
component.set_output_types(self, values=list_type_)
else:
component.set_output_types(self, values=List[Any])
component.set_output_types(self, values=list[Any])
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -90,7 +89,7 @@ class ListJoiner:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ListJoiner":
def from_dict(cls, data: dict[str, Any]) -> "ListJoiner":
"""
Deserializes the component from a dictionary.
@@ -102,7 +101,7 @@ class ListJoiner:
data["init_parameters"]["list_type_"] = deserialize_type(data["init_parameters"]["list_type_"])
return default_from_dict(cls, data)
def run(self, values: Variadic[List[Any]]) -> Dict[str, List[Any]]:
def run(self, values: Variadic[list[Any]]) -> dict[str, list[Any]]:
"""
Joins multiple lists into a single flat list.
+1 -2
View File
@@ -2,7 +2,6 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import List
from haystack import component
from haystack.core.component.types import Variadic
@@ -40,7 +39,7 @@ class StringJoiner:
```
"""
@component.output_types(strings=List[str])
@component.output_types(strings=list[str])
def run(self, strings: Variadic[str]):
"""
Joins strings into a list of strings
@@ -4,7 +4,7 @@
from copy import deepcopy
from io import StringIO
from typing import Dict, List, Optional
from typing import Optional
from haystack import Document, component, logging
from haystack.lazy_imports import LazyImport
@@ -54,8 +54,8 @@ class CSVDocumentCleaner:
self.keep_id = keep_id
pandas_import.check()
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
"""
Cleans CSV documents by removing empty rows and columns while preserving specified ignored rows and columns.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from io import StringIO
from typing import Any, Dict, List, Literal, Optional, Tuple, get_args
from typing import Any, Literal, Optional, get_args
from haystack import Document, component, logging
from haystack.lazy_imports import LazyImport
@@ -32,7 +32,7 @@ class CSVDocumentSplitter:
self,
row_split_threshold: Optional[int] = 2,
column_split_threshold: Optional[int] = 2,
read_csv_kwargs: Optional[Dict[str, Any]] = None,
read_csv_kwargs: Optional[dict[str, Any]] = None,
split_mode: SplitMode = "threshold",
) -> None:
"""
@@ -70,8 +70,8 @@ class CSVDocumentSplitter:
self.read_csv_kwargs = read_csv_kwargs or {}
self.split_mode = split_mode
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
"""
Processes and splits a list of CSV documents into multiple sub-tables.
@@ -162,7 +162,7 @@ class CSVDocumentSplitter:
@staticmethod
def _find_split_indices(
df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
) -> List[Tuple[int, int]]:
) -> list[tuple[int, int]]:
"""
Finds the indices of consecutive empty rows or columns in a DataFrame.
@@ -202,7 +202,7 @@ class CSVDocumentSplitter:
def _split_dataframe(
self, df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
) -> List["pd.DataFrame"]:
) -> list["pd.DataFrame"]:
"""
Splits a DataFrame into sub-tables based on consecutive empty rows or columns exceeding `split_threshold`.
@@ -237,7 +237,7 @@ class CSVDocumentSplitter:
def _recursive_split(
self, df: "pd.DataFrame", row_split_threshold: int, column_split_threshold: int
) -> List["pd.DataFrame"]:
) -> list["pd.DataFrame"]:
"""
Recursively splits a DataFrame.
@@ -272,7 +272,7 @@ class CSVDocumentSplitter:
return result
def _split_by_row(self, df: "pd.DataFrame") -> List["pd.DataFrame"]:
def _split_by_row(self, df: "pd.DataFrame") -> list["pd.DataFrame"]:
"""Split each CSV row into a separate subtable"""
split_dfs = []
for idx, row in enumerate(df.itertuples(index=False)):
@@ -6,7 +6,7 @@ import re
from copy import deepcopy
from functools import partial, reduce
from itertools import chain
from typing import Generator, List, Literal, Optional, Set
from typing import Generator, Literal, Optional
from unicodedata import normalize
from haystack import Document, component, logging
@@ -44,7 +44,7 @@ class DocumentCleaner:
remove_extra_whitespaces: bool = True,
remove_repeated_substrings: bool = False,
keep_id: bool = False,
remove_substrings: Optional[List[str]] = None,
remove_substrings: Optional[list[str]] = None,
remove_regex: Optional[str] = None,
unicode_normalization: Optional[Literal["NFC", "NFKC", "NFD", "NFKD"]] = None,
ascii_only: bool = False,
@@ -89,8 +89,8 @@ class DocumentCleaner:
if unicode_normalization and unicode_normalization not in ["NFC", "NFKC", "NFD", "NFKD"]:
raise ValueError("unicode_normalization must be one of 'NFC', 'NFKC', 'NFD', 'NFKD'.")
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Cleans up the documents.
@@ -204,7 +204,7 @@ class DocumentCleaner:
cleaned_text = [re.sub(regex, "", text).strip() for text in texts]
return "\f".join(cleaned_text)
def _remove_substrings(self, text: str, substrings: List[str]) -> str:
def _remove_substrings(self, text: str, substrings: list[str]) -> str:
"""
Remove all specified substrings from the text.
@@ -287,7 +287,7 @@ class DocumentCleaner:
return ngrams
def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> Set[str]:
def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> set[str]:
"""
Generates all possible ngrams from a given sequence of text.
@@ -303,7 +303,7 @@ class DocumentCleaner:
res = set(chain.from_iterable(ngrams))
return res
def _find_longest_common_ngram(self, sequences: List[str], min_ngram: int = 3, max_ngram: int = 30) -> str:
def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, max_ngram: int = 30) -> str:
"""
Find the longest common ngram across a list of text sequences (e.g. start of pages).
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional
from haystack import Document, Pipeline, default_from_dict, default_to_dict, super_component
from haystack.components.preprocessors.document_cleaner import DocumentCleaner
@@ -38,7 +38,7 @@ class DocumentPreprocessor:
split_length: int = 250,
split_overlap: int = 0,
split_threshold: int = 0,
splitting_function: Optional[Callable[[str], List[str]]] = None,
splitting_function: Optional[Callable[[str], list[str]]] = None,
respect_sentence_boundary: bool = False,
language: Language = "en",
use_split_rules: bool = True,
@@ -48,7 +48,7 @@ class DocumentPreprocessor:
remove_extra_whitespaces: bool = True,
remove_repeated_substrings: bool = False,
keep_id: bool = False,
remove_substrings: Optional[List[str]] = None,
remove_substrings: Optional[list[str]] = None,
remove_regex: Optional[str] = None,
unicode_normalization: Optional[Literal["NFC", "NFKC", "NFD", "NFKD"]] = None,
ascii_only: bool = False,
@@ -144,12 +144,12 @@ class DocumentPreprocessor:
if TYPE_CHECKING:
# fake method, never executed, but static analyzers will not complain about missing method
def run(self, *, documents: List[Document]) -> dict[str, list[Document]]: # noqa: D102
def run(self, *, documents: list[Document]) -> dict[str, list[Document]]: # noqa: D102
...
def warm_up(self) -> None: # noqa: D102
...
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serialize SuperComponent to a dictionary.
@@ -182,7 +182,7 @@ class DocumentPreprocessor:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DocumentPreprocessor":
def from_dict(cls, data: dict[str, Any]) -> "DocumentPreprocessor":
"""
Deserializes the SuperComponent from a dictionary.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from copy import deepcopy
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
from typing import Any, Callable, Literal, Optional
from more_itertools import windowed
@@ -57,7 +57,7 @@ class DocumentSplitter:
split_length: int = 200,
split_overlap: int = 0,
split_threshold: int = 0,
splitting_function: Optional[Callable[[str], List[str]]] = None,
splitting_function: Optional[Callable[[str], list[str]]] = None,
respect_sentence_boundary: bool = False,
language: Language = "en",
use_split_rules: bool = True,
@@ -168,8 +168,8 @@ class DocumentSplitter:
keep_white_spaces=True,
)
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Split documents into smaller parts.
@@ -194,7 +194,7 @@ class DocumentSplitter:
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
raise TypeError("DocumentSplitter expects a List of Documents as input.")
split_docs: List[Document] = []
split_docs: list[Document] = []
for doc in documents:
if doc.content is None:
raise ValueError(
@@ -207,7 +207,7 @@ class DocumentSplitter:
split_docs += self._split_document(doc)
return {"documents": split_docs}
def _split_document(self, doc: Document) -> List[Document]:
def _split_document(self, doc: Document) -> list[Document]:
if self.split_by == "sentence" or self.respect_sentence_boundary:
return self._split_by_nltk_sentence(doc)
@@ -216,7 +216,7 @@ class DocumentSplitter:
return self._split_by_character(doc)
def _split_by_nltk_sentence(self, doc: Document) -> List[Document]:
def _split_by_nltk_sentence(self, doc: Document) -> list[Document]:
split_docs = []
result = self.sentence_splitter.split_sentences(doc.content) # type: ignore # None check is done in run()
@@ -241,7 +241,7 @@ class DocumentSplitter:
return split_docs
def _split_by_character(self, doc) -> List[Document]:
def _split_by_character(self, doc) -> list[Document]:
split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
units = doc.content.split(split_at)
# Add the delimiter back to all units except the last one
@@ -256,10 +256,10 @@ class DocumentSplitter:
text_splits=text_splits, splits_pages=splits_pages, splits_start_idxs=splits_start_idxs, meta=metadata
)
def _split_by_function(self, doc) -> List[Document]:
def _split_by_function(self, doc) -> list[Document]:
# the check for None is done already in the run method
splits = self.splitting_function(doc.content) # type: ignore
docs: List[Document] = []
docs: list[Document] = []
for s in splits:
meta = deepcopy(doc.meta)
meta["source_id"] = doc.id
@@ -267,8 +267,8 @@ class DocumentSplitter:
return docs
def _concatenate_units(
self, elements: List[str], split_length: int, split_overlap: int, split_threshold: int
) -> Tuple[List[str], List[int], List[int]]:
self, elements: list[str], split_length: int, split_overlap: int, split_threshold: int
) -> tuple[list[str], list[int], list[int]]:
"""
Concatenates the elements into parts of split_length units.
@@ -277,9 +277,9 @@ class DocumentSplitter:
units with the last split, preventing the creation of excessively small splits.
"""
text_splits: List[str] = []
splits_pages: List[int] = []
splits_start_idxs: List[int] = []
text_splits: list[str] = []
splits_pages: list[int] = []
splits_start_idxs: list[int] = []
cur_start_idx = 0
cur_page = 1
segments = windowed(elements, n=split_length, step=split_length - split_overlap)
@@ -312,12 +312,12 @@ class DocumentSplitter:
return text_splits, splits_pages, splits_start_idxs
def _create_docs_from_splits(
self, text_splits: List[str], splits_pages: List[int], splits_start_idxs: List[int], meta: Dict[str, Any]
) -> List[Document]:
self, text_splits: list[str], splits_pages: list[int], splits_start_idxs: list[int], meta: dict[str, Any]
) -> list[Document]:
"""
Creates Document objects from splits enriching them with page number and the metadata of the original document.
"""
documents: List[Document] = []
documents: list[Document] = []
for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs)):
copied_meta = deepcopy(meta)
@@ -367,7 +367,7 @@ class DocumentSplitter:
overlapping_range = (0, overlapping_range[1] - overlapping_range[0])
previous_doc.meta["_split_overlap"].append({"doc_id": current_doc.id, "range": overlapping_range})
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
"""
@@ -388,7 +388,7 @@ class DocumentSplitter:
return serialized
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "DocumentSplitter":
def from_dict(cls, data: dict[str, Any]) -> "DocumentSplitter":
"""
Deserializes the component from a dictionary.
"""
@@ -402,8 +402,8 @@ class DocumentSplitter:
@staticmethod
def _concatenate_sentences_based_on_word_amount(
sentences: List[str], split_length: int, split_overlap: int
) -> Tuple[List[str], List[int], List[int]]:
sentences: list[str], split_length: int, split_overlap: int
) -> tuple[list[str], list[int], list[int]]:
"""
Groups the sentences into chunks of `split_length` words while respecting sentence boundaries.
@@ -419,10 +419,10 @@ class DocumentSplitter:
chunk_word_count = 0
chunk_starting_page_number = 1
chunk_start_idx = 0
current_chunk: List[str] = []
current_chunk: list[str] = []
# output lists
split_start_page_numbers = []
list_of_splits: List[List[str]] = []
list_of_splits: list[list[str]] = []
split_start_indices = []
for sentence_idx, sentence in enumerate(sentences):
@@ -470,7 +470,7 @@ class DocumentSplitter:
return text_splits, split_start_page_numbers, split_start_indices
@staticmethod
def _number_of_sentences_to_keep(sentences: List[str], split_length: int, split_overlap: int) -> int:
def _number_of_sentences_to_keep(sentences: list[str], split_length: int, split_overlap: int) -> int:
"""
Returns the number of sentences to keep in the next chunk based on the `split_overlap` and `split_length`.
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Literal, Set
from typing import Any, Literal
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.components.preprocessors import DocumentSplitter
@@ -36,7 +36,7 @@ class HierarchicalDocumentSplitter:
def __init__(
self,
block_sizes: Set[int],
block_sizes: set[int],
split_overlap: int = 0,
split_by: Literal["word", "sentence", "page", "passage"] = "word",
):
@@ -49,13 +49,13 @@ class HierarchicalDocumentSplitter:
"""
self.block_sizes = sorted(set(block_sizes), reverse=True)
self.splitters: Dict[int, DocumentSplitter] = {}
self.splitters: dict[int, DocumentSplitter] = {}
self.split_overlap = split_overlap
self.split_by = split_by
self._build_block_sizes()
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]):
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]):
"""
Builds a hierarchical document structure for each document in a list of documents.
@@ -82,7 +82,7 @@ class HierarchicalDocumentSplitter:
document.meta["__level"] = 0
return document
def build_hierarchy_from_doc(self, document: Document) -> List[Document]:
def build_hierarchy_from_doc(self, document: Document) -> list[Document]:
"""
Build a hierarchical tree document structure from a single document.
@@ -119,7 +119,7 @@ class HierarchicalDocumentSplitter:
return [root] + all_docs
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Returns a dictionary representation of the component.
@@ -131,7 +131,7 @@ class HierarchicalDocumentSplitter:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HierarchicalDocumentSplitter":
def from_dict(cls, data: dict[str, Any]) -> "HierarchicalDocumentSplitter":
"""
Deserialize this component from a dictionary.
@@ -4,7 +4,7 @@
import re
from copy import deepcopy
from typing import Any, Dict, List, Literal, Optional, Tuple
from typing import Any, Literal, Optional
from haystack import Document, component, logging
from haystack.lazy_imports import LazyImport
@@ -62,8 +62,8 @@ class RecursiveDocumentSplitter:
split_length: int = 200,
split_overlap: int = 0,
split_unit: Literal["word", "char", "token"] = "word",
separators: Optional[List[str]] = None,
sentence_splitter_params: Optional[Dict[str, Any]] = None,
separators: Optional[list[str]] = None,
sentence_splitter_params: Optional[dict[str, Any]] = None,
):
"""
Initializes a RecursiveDocumentSplitter.
@@ -118,12 +118,12 @@ class RecursiveDocumentSplitter:
raise ValueError("All separators must be strings.")
@staticmethod
def _get_custom_sentence_tokenizer(sentence_splitter_params: Dict[str, Any]):
def _get_custom_sentence_tokenizer(sentence_splitter_params: dict[str, Any]):
from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
return SentenceSplitter(**sentence_splitter_params)
def _split_chunk(self, current_chunk: str) -> Tuple[str, str]:
def _split_chunk(self, current_chunk: str) -> tuple[str, str]:
"""
Splits a chunk based on the split_length and split_units attribute.
@@ -148,7 +148,7 @@ class RecursiveDocumentSplitter:
remaining_tokens = tokens[self.split_length :]
return self.tiktoken_tokenizer.decode(current_tokens), self.tiktoken_tokenizer.decode(remaining_tokens) # type: ignore
def _apply_overlap(self, chunks: List[str]) -> List[str]:
def _apply_overlap(self, chunks: list[str]) -> list[str]:
"""
Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
@@ -160,7 +160,7 @@ class RecursiveDocumentSplitter:
:returns:
A list of text chunks with the overlap applied.
"""
overlapped_chunks: List[str] = []
overlapped_chunks: list[str] = []
for idx, chunk in enumerate(chunks):
if idx == 0:
@@ -238,7 +238,7 @@ class RecursiveDocumentSplitter:
current_chunk = overlap + chunk
return current_chunk
def _get_overlap(self, overlapped_chunks: List[str]) -> Tuple[str, str]:
def _get_overlap(self, overlapped_chunks: list[str]) -> tuple[str, str]:
"""Get the previous overlapped chunk instead of the original chunk."""
prev_chunk = overlapped_chunks[-1]
overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
@@ -273,7 +273,7 @@ class RecursiveDocumentSplitter:
# at this point we know that the tokenizer is already initialized
return len(self.tiktoken_tokenizer.encode(text)) # type: ignore
def _chunk_text(self, text: str) -> List[str]:
def _chunk_text(self, text: str) -> list[str]:
"""
Recursive chunking algorithm that divides text into smaller chunks based on a list of separator characters.
@@ -313,7 +313,7 @@ class RecursiveDocumentSplitter:
continue
chunks = []
current_chunk: List[str] = []
current_chunk: list[str] = []
current_length = 0
# check splits, if any is too long, recursively chunk it, otherwise add to current chunk
@@ -356,7 +356,7 @@ class RecursiveDocumentSplitter:
# if no separator worked, fall back to word- or character-level chunking
return self._fall_back_to_fixed_chunking(text, self.split_units)
def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> List[str]:
def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> list[str]:
"""
Fall back to a fixed chunking approach if no separator works for the text.
@@ -399,7 +399,7 @@ class RecursiveDocumentSplitter:
chunks.append(self.tiktoken_tokenizer.decode(chunk_tokens)) # type: ignore
return chunks
def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: List[Document]) -> None:
def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None:
prev_doc = new_docs[-1]
overlap_length = self._chunk_length(prev_doc.content) - (curr_pos - prev_doc.meta["split_idx_start"]) # type: ignore
if overlap_length > 0:
@@ -414,13 +414,13 @@ class RecursiveDocumentSplitter:
}
)
def _run_one(self, doc: Document) -> List[Document]:
def _run_one(self, doc: Document) -> list[Document]:
chunks = self._chunk_text(doc.content) # type: ignore # the caller already check for a non-empty doc.content
chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks # remove last empty chunk if it exists
current_position = 0
current_page = 1
new_docs: List[Document] = []
new_docs: list[Document] = []
for split_nr, chunk in enumerate(chunks):
meta = deepcopy(doc.meta)
@@ -452,8 +452,8 @@ class RecursiveDocumentSplitter:
return new_docs
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
"""
Split a list of documents into documents with smaller chunks of text.
@@ -4,7 +4,7 @@
import re
from pathlib import Path
from typing import Any, Dict, List, Literal, Tuple
from typing import Any, Literal
from haystack import logging
from haystack.lazy_imports import LazyImport
@@ -144,7 +144,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
self.sentence_tokenizer._params.abbrev_types.update(abbreviations)
self.keep_white_spaces = keep_white_spaces
def split_sentences(self, text: str) -> List[Dict[str, Any]]:
def split_sentences(self, text: str) -> list[dict[str, Any]]:
"""
Splits a text into sentences including references to original char positions for each split.
@@ -159,7 +159,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
return sentences
@staticmethod
def _apply_split_rules(text: str, sentence_spans: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
def _apply_split_rules(text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
"""
Applies additional split rules to the sentence spans.
@@ -182,7 +182,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
@staticmethod
def _needs_join(
text: str, span: Tuple[int, int], next_span: Tuple[int, int], quote_spans: List[Tuple[int, int]]
text: str, span: tuple[int, int], next_span: tuple[int, int], quote_spans: list[tuple[int, int]]
) -> bool:
"""
Checks if the spans need to be joined as parts of one sentence.
@@ -222,7 +222,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
return re.search(r"^\s*[\(\[]", text[next_start:next_end]) is not None
@staticmethod
def _read_abbreviations(lang: Language) -> List[str]:
def _read_abbreviations(lang: Language) -> list[str]:
"""
Reads the abbreviations for a given language from the abbreviations file.
@@ -4,7 +4,7 @@
import re
import string
from typing import Any, Dict, List, Optional
from typing import Any, Optional
from haystack import component
@@ -32,7 +32,7 @@ class TextCleaner:
def __init__(
self,
remove_regexps: Optional[List[str]] = None,
remove_regexps: Optional[list[str]] = None,
convert_to_lowercase: bool = False,
remove_punctuation: bool = False,
remove_numbers: bool = False,
@@ -61,8 +61,8 @@ class TextCleaner:
self._translator = str.maketrans("", "", to_remove) if to_remove else None
@component.output_types(texts=List[str])
def run(self, texts: List[str]) -> Dict[str, Any]:
@component.output_types(texts=list[str])
def run(self, texts: list[str]) -> dict[str, Any]:
"""
Cleans up the given list of strings.
+16 -16
View File
@@ -4,7 +4,7 @@
import copy
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from urllib.parse import urljoin
from haystack import Document, component, default_from_dict, default_to_dict
@@ -66,7 +66,7 @@ class HuggingFaceTEIRanker:
raw_scores: bool = False,
timeout: Optional[int] = 30,
max_retries: int = 3,
retry_status_codes: Optional[List[int]] = None,
retry_status_codes: Optional[list[int]] = None,
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
) -> None:
"""
@@ -91,7 +91,7 @@ class HuggingFaceTEIRanker:
self.retry_status_codes = retry_status_codes
self.raw_scores = raw_scores
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -109,7 +109,7 @@ class HuggingFaceTEIRanker:
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceTEIRanker":
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceTEIRanker":
"""
Deserializes the component from a dictionary.
@@ -122,8 +122,8 @@ class HuggingFaceTEIRanker:
return default_from_dict(cls, data)
def _compose_response(
self, result: Union[Dict[str, str], List[Dict[str, Any]]], top_k: Optional[int], documents: List[Document]
) -> Dict[str, List[Document]]:
self, result: Union[dict[str, str], list[dict[str, Any]]], top_k: Optional[int], documents: list[Document]
) -> dict[str, list[Document]]:
"""
Processes the API response into a structured format.
@@ -162,14 +162,14 @@ class HuggingFaceTEIRanker:
ranked_docs.append(doc_copy)
return {"documents": ranked_docs}
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
truncation_direction: Optional[TruncationDirection] = None,
) -> Dict[str, List[Document]]:
) -> dict[str, list[Document]]:
"""
Reranks the provided documents by relevance to the query using the TEI API.
@@ -193,7 +193,7 @@ class HuggingFaceTEIRanker:
# Prepare the payload
texts = [doc.content for doc in documents]
payload: Dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
payload: dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
if truncation_direction:
payload.update({"truncate": True, "truncation_direction": truncation_direction.value})
@@ -212,18 +212,18 @@ class HuggingFaceTEIRanker:
status_codes_to_retry=self.retry_status_codes,
)
result: Union[Dict[str, str], List[Dict[str, Any]]] = response.json()
result: Union[dict[str, str], list[dict[str, Any]]] = response.json()
return self._compose_response(result, top_k, documents)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
async def run_async(
self,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
truncation_direction: Optional[TruncationDirection] = None,
) -> Dict[str, List[Document]]:
) -> dict[str, list[Document]]:
"""
Asynchronously reranks the provided documents by relevance to the query using the TEI API.
@@ -246,7 +246,7 @@ class HuggingFaceTEIRanker:
# Prepare the payload
texts = [doc.content for doc in documents]
payload: Dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
payload: dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
if truncation_direction:
payload.update({"truncate": True, "truncation_direction": truncation_direction.value})
@@ -265,6 +265,6 @@ class HuggingFaceTEIRanker:
status_codes_to_retry=self.retry_status_codes,
)
result: Union[Dict[str, str], List[Dict[str, Any]]] = response.json()
result: Union[dict[str, str], list[dict[str, Any]]] = response.json()
return self._compose_response(result, top_k, documents)
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0
from typing import Dict, List, Optional
from typing import Optional
from haystack import Document, component
@@ -59,10 +59,10 @@ class LostInTheMiddleRanker:
self.word_count_threshold = word_count_threshold
self.top_k = top_k
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self, documents: List[Document], top_k: Optional[int] = None, word_count_threshold: Optional[int] = None
) -> Dict[str, List[Document]]:
self, documents: list[Document], top_k: Optional[int] = None, word_count_threshold: Optional[int] = None
) -> dict[str, list[Document]]:
"""
Reranks documents based on the "lost in the middle" order.
+9 -9
View File
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from collections import defaultdict
from typing import Any, Callable, Dict, List, Literal, Optional
from typing import Any, Callable, Literal, Optional
from dateutil.parser import parse as date_parse
@@ -156,10 +156,10 @@ class MetaFieldRanker:
"MetaFieldRanker." % meta_value_type
)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run( # pylint: disable=too-many-positional-arguments
self,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
weight: Optional[float] = None,
ranking_mode: Optional[Literal["reciprocal_rank_fusion", "linear_score"]] = None,
@@ -320,8 +320,8 @@ class MetaFieldRanker:
return {"documents": sorted_documents[:top_k]}
def _parse_meta(
self, docs_with_meta_field: List[Document], meta_value_type: Optional[Literal["float", "int", "date"]]
) -> List[Any]:
self, docs_with_meta_field: list[Document], meta_value_type: Optional[Literal["float", "int", "date"]]
) -> list[Any]:
"""
Parse the meta values stored under `self.meta_field` for the Documents provided in `docs_with_meta_field`.
"""
@@ -364,15 +364,15 @@ class MetaFieldRanker:
def _merge_rankings(
self,
documents: List[Document],
sorted_documents: List[Document],
documents: list[Document],
sorted_documents: list[Document],
weight: float,
ranking_mode: Literal["reciprocal_rank_fusion", "linear_score"],
) -> List[Document]:
) -> list[Document]:
"""
Merge the two different rankings for Documents sorted both by their content and by their meta field.
"""
scores_map: Dict = defaultdict(int)
scores_map: dict = defaultdict(int)
if ranking_mode == "reciprocal_rank_fusion":
for i, (document, sorted_doc) in enumerate(zip(documents, sorted_documents)):
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from collections import defaultdict
from typing import Any, Dict, List, Optional, cast
from typing import Any, Optional, cast
from haystack import Document, component
@@ -72,8 +72,8 @@ class MetaFieldGroupingRanker:
self.sort_docs_by = sort_docs_by
self.subgroup_by = subgroup_by
@component.output_types(documents=List[Document])
def run(self, documents: List[Document]) -> Dict[str, Any]:
@component.output_types(documents=list[Document])
def run(self, documents: list[Document]) -> dict[str, Any]:
"""
Groups the provided list of documents based on the `group_by` parameter and optionally the `subgroup_by`.
@@ -88,7 +88,7 @@ class MetaFieldGroupingRanker:
if not documents:
return {"documents": []}
document_groups: Dict[str, Dict[str, List[Document]]] = defaultdict(lambda: defaultdict(list))
document_groups: dict[str, dict[str, list[Document]]] = defaultdict(lambda: defaultdict(list))
no_group_docs = []
for doc in documents:
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Literal, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.lazy_imports import LazyImport
@@ -120,13 +120,13 @@ class SentenceTransformersDiversityRanker:
query_suffix: str = "",
document_prefix: str = "",
document_suffix: str = "",
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
strategy: Union[str, DiversityRankingStrategy] = "greedy_diversity_order",
lambda_threshold: float = 0.5,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
):
"""
@@ -206,7 +206,7 @@ class SentenceTransformersDiversityRanker:
backend=self.backend,
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -238,7 +238,7 @@ class SentenceTransformersDiversityRanker:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDiversityRanker":
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDiversityRanker":
"""
Deserializes the component from a dictionary.
@@ -255,7 +255,7 @@ class SentenceTransformersDiversityRanker:
deserialize_hf_model_kwargs(init_params["model_kwargs"])
return default_from_dict(cls, data)
def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
"""
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
"""
@@ -273,7 +273,7 @@ class SentenceTransformersDiversityRanker:
return texts_to_embed
def _greedy_diversity_order(self, query: str, documents: List[Document]) -> List[Document]:
def _greedy_diversity_order(self, query: str, documents: list[Document]) -> list[Document]:
"""
Orders the given list of documents to maximize diversity.
@@ -293,7 +293,7 @@ class SentenceTransformersDiversityRanker:
doc_embeddings, query_embedding = self._embed_and_normalize(query, texts_to_embed)
n = len(documents)
selected: List[int] = []
selected: list[int] = []
# Compute the similarity vector between the query and documents
query_doc_sim = query_embedding @ doc_embeddings.T
@@ -315,7 +315,7 @@ class SentenceTransformersDiversityRanker:
# It's divided by n for numerical stability
selected_sum += doc_embeddings[index_unselected] / n
ranked_docs: List[Document] = [documents[i] for i in selected]
ranked_docs: list[Document] = [documents[i] for i in selected]
return ranked_docs
@@ -333,8 +333,8 @@ class SentenceTransformersDiversityRanker:
return doc_embeddings, query_embedding
def _maximum_margin_relevance(
self, query: str, documents: List[Document], lambda_threshold: float, top_k: int
) -> List[Document]:
self, query: str, documents: list[Document], lambda_threshold: float, top_k: int
) -> list[Document]:
"""
Orders the given list of documents according to the Maximum Margin Relevance (MMR) scores.
@@ -355,7 +355,7 @@ class SentenceTransformersDiversityRanker:
doc_embeddings, query_embedding = self._embed_and_normalize(query, texts_to_embed)
top_k = top_k if top_k else len(documents)
selected: List[int] = []
selected: list[int] = []
query_similarities_as_tensor = query_embedding @ doc_embeddings.T
query_similarities = query_similarities_as_tensor.reshape(-1)
idx = int(torch.argmax(query_similarities))
@@ -386,14 +386,14 @@ class SentenceTransformersDiversityRanker:
if (strategy == DiversityRankingStrategy.MAXIMUM_MARGIN_RELEVANCE) and not 0 <= lambda_threshold <= 1:
raise ValueError(f"lambda_threshold must be between 0 and 1, but got {lambda_threshold}.")
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
lambda_threshold: Optional[float] = None,
) -> Dict[str, List[Document]]:
) -> dict[str, list[Document]]:
"""
Rank the documents based on their diversity.
@@ -4,7 +4,7 @@
from copy import copy
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Literal, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict
from haystack.lazy_imports import LazyImport
@@ -48,14 +48,14 @@ class SentenceTransformersSimilarityRanker:
top_k: int = 10,
query_prefix: str = "",
document_prefix: str = "",
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
scale_score: bool = True,
score_threshold: Optional[float] = None,
trust_remote_code: bool = False,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
config_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
config_kwargs: Optional[dict[str, Any]] = None,
backend: Literal["torch", "onnx", "openvino"] = "torch",
batch_size: int = 16,
):
@@ -130,7 +130,7 @@ class SentenceTransformersSimilarityRanker:
self.backend = backend
self.batch_size = batch_size
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
@@ -152,7 +152,7 @@ class SentenceTransformersSimilarityRanker:
backend=self.backend,
)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -183,7 +183,7 @@ class SentenceTransformersSimilarityRanker:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersSimilarityRanker":
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersSimilarityRanker":
"""
Deserializes the component from a dictionary.
@@ -201,16 +201,16 @@ class SentenceTransformersSimilarityRanker:
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run(
self,
*,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
scale_score: Optional[bool] = None,
score_threshold: Optional[float] = None,
) -> Dict[str, List[Document]]:
) -> dict[str, list[Document]]:
"""
Returns a list of documents ranked by their similarity to the given query.
@@ -3,7 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from haystack import Document, component, default_from_dict, default_to_dict, logging
from haystack.lazy_imports import LazyImport
@@ -56,13 +56,13 @@ class TransformersSimilarityRanker:
top_k: int = 10,
query_prefix: str = "",
document_prefix: str = "",
meta_fields_to_embed: Optional[List[str]] = None,
meta_fields_to_embed: Optional[list[str]] = None,
embedding_separator: str = "\n",
scale_score: bool = True,
calibration_factor: Optional[float] = 1.0,
score_threshold: Optional[float] = None,
model_kwargs: Optional[Dict[str, Any]] = None,
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
tokenizer_kwargs: Optional[dict[str, Any]] = None,
batch_size: int = 16,
):
"""
@@ -146,7 +146,7 @@ class TransformersSimilarityRanker:
if self.top_k <= 0:
raise ValueError(f"top_k must be > 0, but got {top_k}")
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
@@ -168,7 +168,7 @@ class TransformersSimilarityRanker:
assert self.model is not None
self.device = ComponentDevice.from_multiple(device_map=DeviceMap.from_hf(self.model.hf_device_map))
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -197,7 +197,7 @@ class TransformersSimilarityRanker:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "TransformersSimilarityRanker":
def from_dict(cls, data: dict[str, Any]) -> "TransformersSimilarityRanker":
"""
Deserializes the component from a dictionary.
@@ -215,11 +215,11 @@ class TransformersSimilarityRanker:
return default_from_dict(cls, data)
@component.output_types(documents=List[Document])
@component.output_types(documents=list[Document])
def run( # pylint: disable=too-many-positional-arguments
self,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
scale_score: Optional[bool] = None,
calibration_factor: Optional[float] = None,
+23 -23
View File
@@ -4,7 +4,7 @@
import math
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Optional, Union
from haystack import Document, ExtractedAnswer, component, default_from_dict, default_to_dict, logging
from haystack.lazy_imports import LazyImport
@@ -64,7 +64,7 @@ class ExtractiveReader:
no_answer: bool = True,
calibration_factor: float = 0.1,
overlap_threshold: Optional[float] = 0.01,
model_kwargs: Optional[Dict[str, Any]] = None,
model_kwargs: Optional[dict[str, Any]] = None,
) -> None:
"""
Creates an instance of ExtractiveReader.
@@ -126,13 +126,13 @@ class ExtractiveReader:
model_kwargs = resolve_hf_device_map(device=device, model_kwargs=model_kwargs)
self.model_kwargs = model_kwargs
def _get_telemetry_data(self) -> Dict[str, Any]:
def _get_telemetry_data(self) -> dict[str, Any]:
"""
Data that is sent to Posthog for usage analytics.
"""
return {"model": self.model_name_or_path}
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
"""
Serializes the component to a dictionary.
@@ -159,7 +159,7 @@ class ExtractiveReader:
return serialization_dict
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ExtractiveReader":
def from_dict(cls, data: dict[str, Any]) -> "ExtractiveReader":
"""
Deserializes the component from a dictionary.
@@ -194,8 +194,8 @@ class ExtractiveReader:
@staticmethod
def _flatten_documents(
queries: List[str], documents: List[List[Document]]
) -> Tuple[List[str], List[Document], List[int]]:
queries: list[str], documents: list[list[Document]]
) -> tuple[list[str], list[Document], list[int]]:
"""
Flattens queries and Documents so all query-document pairs are arranged along one batch axis.
"""
@@ -205,8 +205,8 @@ class ExtractiveReader:
return flattened_queries, flattened_documents, query_ids
def _preprocess( # pylint: disable=too-many-positional-arguments
self, *, queries: List[str], documents: List[Document], max_seq_length: int, query_ids: List[int], stride: int
) -> Tuple["torch.Tensor", "torch.Tensor", "torch.Tensor", List["Encoding"], List[int], List[int]]:
self, *, queries: list[str], documents: list[Document], max_seq_length: int, query_ids: list[int], stride: int
) -> tuple["torch.Tensor", "torch.Tensor", "torch.Tensor", list["Encoding"], list[int], list[int]]:
"""
Splits and tokenizes Documents and preserves structures by returning mappings to query and Document IDs.
"""
@@ -264,8 +264,8 @@ class ExtractiveReader:
sequence_ids: "torch.Tensor",
attention_mask: "torch.Tensor",
answers_per_seq: int,
encodings: List["Encoding"],
) -> Tuple[List[List[int]], List[List[int]], "torch.Tensor"]:
encodings: list["Encoding"],
) -> tuple[list[list[int]], list[list[int]], "torch.Tensor"]:
"""
Turns start and end logits into probabilities for each answer span.
@@ -347,19 +347,19 @@ class ExtractiveReader:
def _nest_answers(
self,
*,
start: List[List[int]],
end: List[List[int]],
start: list[list[int]],
end: list[list[int]],
probabilities: "torch.Tensor",
flattened_documents: List[Document],
queries: List[str],
flattened_documents: list[Document],
queries: list[str],
answers_per_seq: int,
top_k: Optional[int],
score_threshold: Optional[float],
query_ids: List[int],
document_ids: List[int],
query_ids: list[int],
document_ids: list[int],
no_answer: bool,
overlap_threshold: Optional[float],
) -> List[List[ExtractedAnswer]]:
) -> list[list[ExtractedAnswer]]:
"""
Reconstructs the nested structure that existed before flattening.
@@ -431,7 +431,7 @@ class ExtractiveReader:
return 0
def _should_keep(
self, candidate_answer: ExtractedAnswer, current_answers: List[ExtractedAnswer], overlap_threshold: float
self, candidate_answer: ExtractedAnswer, current_answers: list[ExtractedAnswer], overlap_threshold: float
) -> bool:
"""
Determines if the answer should be kept based on how much it overlaps with previous answers.
@@ -493,8 +493,8 @@ class ExtractiveReader:
return keep
def deduplicate_by_overlap(
self, answers: List[ExtractedAnswer], overlap_threshold: Optional[float]
) -> List[ExtractedAnswer]:
self, answers: list[ExtractedAnswer], overlap_threshold: Optional[float]
) -> list[ExtractedAnswer]:
"""
De-duplicates overlapping Extractive Answers.
@@ -529,11 +529,11 @@ class ExtractiveReader:
return deduplicated_answers
@component.output_types(answers=List[ExtractedAnswer])
@component.output_types(answers=list[ExtractedAnswer])
def run( # pylint: disable=too-many-positional-arguments
self,
query: str,
documents: List[Document],
documents: list[Document],
top_k: Optional[int] = None,
score_threshold: Optional[float] = None,
max_seq_length: Optional[int] = None,

Some files were not shown because too many files have changed in this diff Show More