mirror of
https://github.com/deepset-ai/haystack.git
synced 2026-10-12 02:12:50 +00:00
chore: adopt PEP 585 type hints (#9678)
* chore(lint): enforce and apply PEP 585 type hinting * Run fmt fixes * Fix all typing imports using some regex * Fix all typing written in string in tests * undo changes in the e2e tests * make e2e test use list instead of List * type fixes * remove type:ignore * pylint * Remove typing from Usage example comments * Remove typing from most of comments * try to fix e2e tests on comm PRs * fix * Add tests typing.List in to adjust test compatiplity - test/components/agents/test_state_class.py - test/components/converters/test_output_adapter.py - test/components/joiners/test_list_joiner.py * simplify pyproject * improve relnote --------- Co-authored-by: anakin87 <[email protected]>
This commit is contained in:
co-authored by
anakin87
parent
4b9fb20bab
commit
5f3c37d287
@@ -3,12 +3,12 @@ import os
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from haystack import logging # pylint: disable=unused-import # this is needed to avoid circular imports
|
||||
|
||||
|
||||
def validate_module_imports(root_dir: str, exclude_subdirs: Optional[List[str]] = None) -> tuple[list, list]:
|
||||
def validate_module_imports(root_dir: str, exclude_subdirs: Optional[list[str]] = None) -> tuple[list, list]:
|
||||
"""
|
||||
Recursively search for all Python modules and attempt to import them.
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ import base64
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
@@ -30,7 +29,7 @@ def create_headers(version: str):
|
||||
return {"authorization": f"Basic {readme_token()}", "x-readme-version": version}
|
||||
|
||||
|
||||
def get_docs_in_category(category_slug: str, version: str) -> List[str]:
|
||||
def get_docs_in_category(category_slug: str, version: str) -> list[str]:
|
||||
"""
|
||||
Returns the slugs of all documents in a category for the specific version.
|
||||
"""
|
||||
|
||||
@@ -15,7 +15,10 @@ from haystack.components.writers import DocumentWriter
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
|
||||
|
||||
def test_dense_doc_search_pipeline(tmp_path, samples_path):
|
||||
def test_dense_doc_search_pipeline(tmp_path, samples_path, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
# Create the indexing pipeline
|
||||
indexing_pipeline = Pipeline()
|
||||
indexing_pipeline.add_component(
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -30,7 +29,7 @@ from haystack.dataclasses import ChatMessage
|
||||
EMBEDDINGS_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
|
||||
|
||||
def indexing_pipeline(documents: List[Document]):
|
||||
def indexing_pipeline(documents: list[Document]):
|
||||
"""Indexing the documents"""
|
||||
document_store = InMemoryDocumentStore()
|
||||
doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP)
|
||||
|
||||
@@ -8,7 +8,10 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
|
||||
|
||||
def test_extractive_qa_pipeline(tmp_path):
|
||||
def test_extractive_qa_pipeline(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
# Create the pipeline
|
||||
qa_pipeline = Pipeline()
|
||||
qa_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=InMemoryDocumentStore()), name="retriever")
|
||||
|
||||
@@ -11,7 +11,10 @@ from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMe
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
|
||||
|
||||
def test_hybrid_doc_search_pipeline(tmp_path):
|
||||
def test_hybrid_doc_search_pipeline(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
# Create the pipeline
|
||||
document_store = InMemoryDocumentStore()
|
||||
hybrid_pipeline = Pipeline()
|
||||
|
||||
@@ -47,7 +47,10 @@ def spacy_annotations():
|
||||
]
|
||||
|
||||
|
||||
def test_ner_extractor_init():
|
||||
def test_ner_extractor_init(monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
extractor = NamedEntityExtractor(backend=NamedEntityExtractorBackend.HUGGING_FACE, model="dslim/bert-base-NER")
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"not warmed up"):
|
||||
@@ -59,7 +62,10 @@ def test_ner_extractor_init():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 3])
|
||||
def test_ner_extractor_hf_backend(raw_texts, hf_annotations, batch_size):
|
||||
def test_ner_extractor_hf_backend(raw_texts, hf_annotations, batch_size, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
extractor = NamedEntityExtractor(backend=NamedEntityExtractorBackend.HUGGING_FACE, model="dslim/bert-base-NER")
|
||||
extractor.warm_up()
|
||||
|
||||
@@ -87,7 +93,10 @@ def test_ner_extractor_spacy_backend(raw_texts, spacy_annotations, batch_size):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 3])
|
||||
def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size):
|
||||
def test_ner_extractor_in_pipeline(raw_texts, hf_annotations, batch_size, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component(
|
||||
name="ner_extractor",
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.converters.pypdf import PyPDFToDocument
|
||||
from haystack.components.joiners import DocumentJoiner
|
||||
@@ -13,6 +17,10 @@ from haystack.components.generators.chat.openai import OpenAIChatGenerator
|
||||
from haystack.components.routers.document_length_router import DocumentLengthRouter
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.environ.get("OPENAI_API_KEY", None),
|
||||
reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
|
||||
)
|
||||
def test_pdf_content_extraction_pipeline():
|
||||
"""
|
||||
Test a pipeline that processes PDFs with the following steps:
|
||||
|
||||
@@ -12,7 +12,10 @@ from haystack.components.writers import DocumentWriter
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
|
||||
|
||||
def test_preprocessing_pipeline(tmp_path):
|
||||
def test_preprocessing_pipeline(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("HF_API_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
monkeypatch.delenv("HF_TOKEN", raising=False) # https://github.com/deepset-ai/haystack/issues/8811
|
||||
|
||||
# Create the pipeline and its components
|
||||
document_store = InMemoryDocumentStore()
|
||||
preprocessing_pipeline = Pipeline()
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import inspect
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import logging, tracing
|
||||
from haystack.components.generators.chat.types import ChatGenerator
|
||||
@@ -71,14 +71,14 @@ class Agent:
|
||||
self,
|
||||
*,
|
||||
chat_generator: ChatGenerator,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
exit_conditions: Optional[List[str]] = None,
|
||||
state_schema: Optional[Dict[str, Any]] = None,
|
||||
exit_conditions: Optional[list[str]] = None,
|
||||
state_schema: Optional[dict[str, Any]] = None,
|
||||
max_agent_steps: int = 100,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
raise_on_tool_invocation_failure: bool = False,
|
||||
tool_invoker_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tool_invoker_kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the agent component.
|
||||
@@ -126,7 +126,7 @@ class Agent:
|
||||
# Initialize state schema
|
||||
resolved_state_schema = _deepcopy_with_exceptions(self._state_schema)
|
||||
if resolved_state_schema.get("messages") is None:
|
||||
resolved_state_schema["messages"] = {"type": List[ChatMessage], "handler": merge_lists}
|
||||
resolved_state_schema["messages"] = {"type": list[ChatMessage], "handler": merge_lists}
|
||||
self.state_schema = resolved_state_schema
|
||||
|
||||
self.chat_generator = chat_generator
|
||||
@@ -172,7 +172,7 @@ class Agent:
|
||||
self.chat_generator.warm_up()
|
||||
self._is_warmed_up = True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize the component to a dictionary.
|
||||
|
||||
@@ -198,7 +198,7 @@ class Agent:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "Agent":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "Agent":
|
||||
"""
|
||||
Deserialize the agent from a dictionary.
|
||||
|
||||
@@ -219,9 +219,9 @@ class Agent:
|
||||
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _prepare_generator_inputs(self, streaming_callback: Optional[StreamingCallbackT] = None) -> Dict[str, Any]:
|
||||
def _prepare_generator_inputs(self, streaming_callback: Optional[StreamingCallbackT] = None) -> dict[str, Any]:
|
||||
"""Prepare inputs for the chat generator."""
|
||||
generator_inputs: Dict[str, Any] = {"tools": self.tools}
|
||||
generator_inputs: dict[str, Any] = {"tools": self.tools}
|
||||
if streaming_callback is not None:
|
||||
generator_inputs["streaming_callback"] = streaming_callback
|
||||
return generator_inputs
|
||||
@@ -240,13 +240,13 @@ class Agent:
|
||||
|
||||
def run( # noqa: PLR0915
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
messages: list[ChatMessage],
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
*,
|
||||
break_point: Optional[AgentBreakpoint] = None,
|
||||
snapshot: Optional[AgentSnapshot] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Process messages and execute tools until an exit condition is met.
|
||||
|
||||
@@ -431,13 +431,13 @@ class Agent:
|
||||
|
||||
async def run_async( # noqa: PLR0915
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
messages: list[ChatMessage],
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
*,
|
||||
break_point: Optional[AgentBreakpoint] = None,
|
||||
snapshot: Optional[AgentSnapshot] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Asynchronously process messages and execute tools until the exit condition is met.
|
||||
|
||||
@@ -626,7 +626,7 @@ class Agent:
|
||||
result.update({"last_message": all_messages[-1]})
|
||||
return result
|
||||
|
||||
def _check_exit_conditions(self, llm_messages: List[ChatMessage], tool_messages: List[ChatMessage]) -> bool:
|
||||
def _check_exit_conditions(self, llm_messages: list[ChatMessage], tool_messages: list[ChatMessage]) -> bool:
|
||||
"""
|
||||
Check if any of the LLM messages' tool calls match an exit condition and if there are no errors.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import Any, Callable, Optional, get_args
|
||||
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.utils import _deserialize_value_with_schema, _serialize_value_with_schema
|
||||
@@ -13,7 +13,7 @@ from haystack.utils.type_serialization import deserialize_type, serialize_type
|
||||
from .state_utils import _is_list_type, _is_valid_type, merge_lists, replace_values
|
||||
|
||||
|
||||
def _schema_to_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _schema_to_dict(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Convert a schema dictionary to a serializable format.
|
||||
|
||||
@@ -32,7 +32,7 @@ def _schema_to_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return serialized_schema
|
||||
|
||||
|
||||
def _schema_from_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _schema_from_dict(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Convert a serialized schema dictionary back to its original format.
|
||||
|
||||
@@ -52,7 +52,7 @@ def _schema_from_dict(schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return deserialized_schema
|
||||
|
||||
|
||||
def _validate_schema(schema: Dict[str, Any]) -> None:
|
||||
def _validate_schema(schema: dict[str, Any]) -> None:
|
||||
"""
|
||||
Validate that a schema dictionary meets all required constraints.
|
||||
|
||||
@@ -69,8 +69,13 @@ def _validate_schema(schema: Dict[str, Any]) -> None:
|
||||
raise ValueError(f"StateSchema: 'type' for key '{param}' must be a Python type, got {definition['type']}")
|
||||
if definition.get("handler") is not None and not callable(definition["handler"]):
|
||||
raise ValueError(f"StateSchema: 'handler' for key '{param}' must be callable or None")
|
||||
if param == "messages" and definition["type"] != List[ChatMessage]:
|
||||
raise ValueError(f"StateSchema: 'messages' must be of type List[ChatMessage], got {definition['type']}")
|
||||
if param == "messages": # definition["type"] != list[ChatMessage] but split to cover also List[ChatMessage]
|
||||
if not _is_list_type(definition["type"]):
|
||||
raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
|
||||
# Check if the list contains ChatMessage elements
|
||||
args = get_args(definition["type"])
|
||||
if not args or args[0] != ChatMessage:
|
||||
raise ValueError(f"StateSchema: 'messages' must be of type list[ChatMessage], got {definition['type']}")
|
||||
|
||||
|
||||
class State:
|
||||
@@ -84,7 +89,7 @@ class State:
|
||||
}
|
||||
"""
|
||||
|
||||
def __init__(self, schema: Dict[str, Any], data: Optional[Dict[str, Any]] = None):
|
||||
def __init__(self, schema: dict[str, Any], data: Optional[dict[str, Any]] = None):
|
||||
"""
|
||||
Initialize a State object with a schema and optional data.
|
||||
|
||||
@@ -98,7 +103,7 @@ class State:
|
||||
_validate_schema(schema)
|
||||
self.schema = deepcopy(schema)
|
||||
if self.schema.get("messages") is None:
|
||||
self.schema["messages"] = {"type": List[ChatMessage], "handler": merge_lists}
|
||||
self.schema["messages"] = {"type": list[ChatMessage], "handler": merge_lists}
|
||||
self._data = data or {}
|
||||
|
||||
# Set default handlers if not provided in schema
|
||||
@@ -170,7 +175,7 @@ class State:
|
||||
return serialized
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]):
|
||||
def from_dict(cls, data: dict[str, Any]):
|
||||
"""
|
||||
Convert a dictionary back to a State object.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import inspect
|
||||
from typing import Any, List, TypeVar, Union, get_origin
|
||||
from typing import Any, TypeVar, Union, get_origin
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -14,7 +14,7 @@ def _is_valid_type(obj: Any) -> bool:
|
||||
|
||||
Valid types include:
|
||||
- Normal classes (str, dict, CustomClass)
|
||||
- Generic types (List[str], Dict[str, int])
|
||||
- Generic types (list[str], dict[str, int])
|
||||
- Union types (Union[str, int], Optional[str])
|
||||
|
||||
:param obj: The object to check
|
||||
@@ -23,7 +23,7 @@ def _is_valid_type(obj: Any) -> bool:
|
||||
Example usage:
|
||||
>>> _is_valid_type(str)
|
||||
True
|
||||
>>> _is_valid_type(List[int])
|
||||
>>> _is_valid_type(list[int])
|
||||
True
|
||||
>>> _is_valid_type(Union[str, int])
|
||||
True
|
||||
@@ -48,7 +48,7 @@ def _is_list_type(type_hint: Any) -> bool:
|
||||
return type_hint == list or (hasattr(type_hint, "__origin__") and get_origin(type_hint) == list)
|
||||
|
||||
|
||||
def merge_lists(current: Union[List[T], T, None], new: Union[List[T], T]) -> List[T]:
|
||||
def merge_lists(current: Union[list[T], T, None], new: Union[list[T], T]) -> list[T]:
|
||||
"""
|
||||
Merges two values into a single list.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Union, get_args
|
||||
from typing import Any, Literal, Optional, Union, get_args
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.dataclasses import ByteStream
|
||||
@@ -54,7 +54,7 @@ class LocalWhisperTranscriber:
|
||||
self,
|
||||
model: WhisperLocalModel = "large",
|
||||
device: Optional[ComponentDevice] = None,
|
||||
whisper_params: Optional[Dict[str, Any]] = None,
|
||||
whisper_params: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an instance of the LocalWhisperTranscriber component.
|
||||
@@ -84,7 +84,7 @@ class LocalWhisperTranscriber:
|
||||
if not self._model:
|
||||
self._model = whisper.load_model(self.model, device=self.device.to_torch())
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -94,7 +94,7 @@ class LocalWhisperTranscriber:
|
||||
return default_to_dict(self, model=self.model, device=self.device.to_dict(), whisper_params=self.whisper_params)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "LocalWhisperTranscriber":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "LocalWhisperTranscriber":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -108,8 +108,8 @@ class LocalWhisperTranscriber:
|
||||
init_params["device"] = ComponentDevice.from_dict(init_params["device"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, sources: List[Union[str, Path, ByteStream]], whisper_params: Optional[Dict[str, Any]] = None):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, sources: list[Union[str, Path, ByteStream]], whisper_params: Optional[dict[str, Any]] = None):
|
||||
"""
|
||||
Transcribes a list of audio files into a list of documents.
|
||||
|
||||
@@ -137,7 +137,7 @@ class LocalWhisperTranscriber:
|
||||
documents = self.transcribe(sources, **whisper_params)
|
||||
return {"documents": documents}
|
||||
|
||||
def transcribe(self, sources: List[Union[str, Path, ByteStream]], **kwargs) -> List[Document]:
|
||||
def transcribe(self, sources: list[Union[str, Path, ByteStream]], **kwargs) -> list[Document]:
|
||||
"""
|
||||
Transcribes the audio files into a list of Documents, one for each input file.
|
||||
|
||||
@@ -158,7 +158,7 @@ class LocalWhisperTranscriber:
|
||||
documents.append(doc)
|
||||
return documents
|
||||
|
||||
def _raw_transcribe(self, sources: List[Union[str, Path, ByteStream]], **kwargs) -> Dict[Path, Any]:
|
||||
def _raw_transcribe(self, sources: list[Union[str, Path, ByteStream]], **kwargs) -> dict[Path, Any]:
|
||||
"""
|
||||
Transcribes the given audio files. Returns the output of the model, a dictionary, for each input file.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
@@ -42,7 +42,7 @@ class RemoteWhisperTranscriber:
|
||||
model: str = "whisper-1",
|
||||
api_base_url: Optional[str] = None,
|
||||
organization: Optional[str] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@@ -105,7 +105,7 @@ class RemoteWhisperTranscriber:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=False),
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -123,7 +123,7 @@ class RemoteWhisperTranscriber:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "RemoteWhisperTranscriber":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "RemoteWhisperTranscriber":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -135,8 +135,8 @@ class RemoteWhisperTranscriber:
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, sources: List[Union[str, Path, ByteStream]]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, sources: list[Union[str, Path, ByteStream]]):
|
||||
"""
|
||||
Transcribes the list of audio files into a list of documents.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, GeneratedAnswer, component, logging
|
||||
from haystack.dataclasses.chat_message import ChatMessage
|
||||
@@ -64,13 +64,13 @@ class AnswerBuilder:
|
||||
self.reference_pattern = reference_pattern
|
||||
self.last_message_only = last_message_only
|
||||
|
||||
@component.output_types(answers=List[GeneratedAnswer])
|
||||
@component.output_types(answers=list[GeneratedAnswer])
|
||||
def run( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
query: str,
|
||||
replies: Union[List[str], List[ChatMessage]],
|
||||
meta: Optional[List[Dict[str, Any]]] = None,
|
||||
documents: Optional[List[Document]] = None,
|
||||
replies: Union[list[str], list[ChatMessage]],
|
||||
meta: Optional[list[dict[str, Any]]] = None,
|
||||
documents: Optional[list[Document]] = None,
|
||||
pattern: Optional[str] = None,
|
||||
reference_pattern: Optional[str] = None,
|
||||
):
|
||||
@@ -184,7 +184,7 @@ class AnswerBuilder:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _extract_reference_idxs(reply: str, reference_pattern: str) -> List[int]:
|
||||
def _extract_reference_idxs(reply: str, reference_pattern: str) -> list[int]:
|
||||
document_idxs = re.findall(reference_pattern, reply)
|
||||
return [int(idx) - 1 for idx in document_idxs]
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from jinja2 import meta
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
@@ -140,9 +140,9 @@ class ChatPromptBuilder:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template: Optional[Union[List[ChatMessage], str]] = None,
|
||||
required_variables: Optional[Union[List[str], Literal["*"]]] = None,
|
||||
variables: Optional[List[str]] = None,
|
||||
template: Optional[Union[list[ChatMessage], str]] = None,
|
||||
required_variables: Optional[Union[list[str], Literal["*"]]] = None,
|
||||
variables: Optional[list[str]] = None,
|
||||
):
|
||||
"""
|
||||
Constructs a ChatPromptBuilder component.
|
||||
@@ -205,11 +205,11 @@ class ChatPromptBuilder:
|
||||
else:
|
||||
component.set_input_type(self, var, Any, "")
|
||||
|
||||
@component.output_types(prompt=List[ChatMessage])
|
||||
@component.output_types(prompt=list[ChatMessage])
|
||||
def run(
|
||||
self,
|
||||
template: Optional[Union[List[ChatMessage], str]] = None,
|
||||
template_variables: Optional[Dict[str, Any]] = None,
|
||||
template: Optional[Union[list[ChatMessage], str]] = None,
|
||||
template_variables: Optional[dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@@ -277,8 +277,8 @@ class ChatPromptBuilder:
|
||||
return {"prompt": processed_messages}
|
||||
|
||||
def _render_chat_messages_from_str_template(
|
||||
self, template: str, template_variables: Dict[str, Any]
|
||||
) -> List[ChatMessage]:
|
||||
self, template: str, template_variables: dict[str, Any]
|
||||
) -> list[ChatMessage]:
|
||||
"""
|
||||
Renders a chat message from a string template.
|
||||
|
||||
@@ -296,7 +296,7 @@ class ChatPromptBuilder:
|
||||
|
||||
return messages
|
||||
|
||||
def _validate_variables(self, provided_variables: Set[str]):
|
||||
def _validate_variables(self, provided_variables: set[str]):
|
||||
"""
|
||||
Checks if all the required template variables are provided.
|
||||
|
||||
@@ -317,14 +317,14 @@ class ChatPromptBuilder:
|
||||
f"Required variables: {required_variables}. Provided variables: {provided_variables}."
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Returns a dictionary representation of the component.
|
||||
|
||||
:returns:
|
||||
Serialized dictionary representation of the component.
|
||||
"""
|
||||
template: Optional[Union[List[Dict[str, Any]], str]] = None
|
||||
template: Optional[Union[list[dict[str, Any]], str]] = None
|
||||
if isinstance(self.template, list):
|
||||
template = [m.to_dict() for m in self.template]
|
||||
elif isinstance(self.template, str):
|
||||
@@ -335,7 +335,7 @@ class ChatPromptBuilder:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ChatPromptBuilder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "ChatPromptBuilder":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from jinja2 import meta
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
@@ -141,8 +141,8 @@ class PromptBuilder:
|
||||
def __init__(
|
||||
self,
|
||||
template: str,
|
||||
required_variables: Optional[Union[List[str], Literal["*"]]] = None,
|
||||
variables: Optional[List[str]] = None,
|
||||
required_variables: Optional[Union[list[str], Literal["*"]]] = None,
|
||||
variables: Optional[list[str]] = None,
|
||||
):
|
||||
"""
|
||||
Constructs a PromptBuilder component.
|
||||
@@ -198,7 +198,7 @@ class PromptBuilder:
|
||||
else:
|
||||
component.set_input_type(self, var, Any, "")
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Returns a dictionary representation of the component.
|
||||
|
||||
@@ -210,7 +210,7 @@ class PromptBuilder:
|
||||
)
|
||||
|
||||
@component.output_types(prompt=str)
|
||||
def run(self, template: Optional[str] = None, template_variables: Optional[Dict[str, Any]] = None, **kwargs):
|
||||
def run(self, template: Optional[str] = None, template_variables: Optional[dict[str, Any]] = None, **kwargs):
|
||||
"""
|
||||
Renders the prompt template with the provided variables.
|
||||
|
||||
@@ -244,7 +244,7 @@ class PromptBuilder:
|
||||
result = compiled_template.render(template_variables_combined)
|
||||
return {"prompt": result}
|
||||
|
||||
def _validate_variables(self, provided_variables: Set[str]):
|
||||
def _validate_variables(self, provided_variables: set[str]):
|
||||
"""
|
||||
Checks if all the required template variables are provided.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.document_stores.types import DocumentStore
|
||||
@@ -51,7 +51,7 @@ class CacheChecker:
|
||||
self.document_store = document_store
|
||||
self.cache_field = cache_field
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -61,7 +61,7 @@ class CacheChecker:
|
||||
return default_to_dict(self, document_store=self.document_store.to_dict(), cache_field=self.cache_field)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "CacheChecker":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "CacheChecker":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -75,8 +75,8 @@ class CacheChecker:
|
||||
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(hits=List[Document], misses=List)
|
||||
def run(self, items: List[Any]):
|
||||
@component.output_types(hits=list[Document], misses=list)
|
||||
def run(self, items: list[Any]):
|
||||
"""
|
||||
Checks if any document associated with the specified cache field is already present in the store.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -52,7 +52,7 @@ class DocumentLanguageClassifier:
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self, languages: Optional[List[str]] = None):
|
||||
def __init__(self, languages: Optional[list[str]] = None):
|
||||
"""
|
||||
Initializes the DocumentLanguageClassifier component.
|
||||
|
||||
@@ -65,8 +65,8 @@ class DocumentLanguageClassifier:
|
||||
languages = ["en"]
|
||||
self.languages = languages
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Classifies the language of each document and adds it to its metadata.
|
||||
|
||||
@@ -86,7 +86,7 @@ class DocumentLanguageClassifier:
|
||||
"In case you want to classify and route a text, please use the TextLanguageRouter."
|
||||
)
|
||||
|
||||
output: Dict[str, List[Document]] = {language: [] for language in self.languages}
|
||||
output: dict[str, list[Document]] = {language: [] for language in self.languages}
|
||||
output["unmatched"] = []
|
||||
|
||||
for document in documents:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -74,12 +74,12 @@ class TransformersZeroShotDocumentClassifier:
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
model: str,
|
||||
labels: List[str],
|
||||
labels: list[str],
|
||||
multi_label: bool = False,
|
||||
classification_field: Optional[str] = None,
|
||||
device: Optional[ComponentDevice] = None,
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
|
||||
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Initializes the TransformersZeroShotDocumentClassifier.
|
||||
@@ -132,7 +132,7 @@ class TransformersZeroShotDocumentClassifier:
|
||||
self.huggingface_pipeline_kwargs = huggingface_pipeline_kwargs
|
||||
self.pipeline: Optional[HfPipeline] = None
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
@@ -147,7 +147,7 @@ class TransformersZeroShotDocumentClassifier:
|
||||
if self.pipeline is None:
|
||||
self.pipeline = pipeline(**self.huggingface_pipeline_kwargs)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -169,7 +169,7 @@ class TransformersZeroShotDocumentClassifier:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "TransformersZeroShotDocumentClassifier":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "TransformersZeroShotDocumentClassifier":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -183,8 +183,8 @@ class TransformersZeroShotDocumentClassifier:
|
||||
deserialize_hf_model_kwargs(data["init_parameters"]["huggingface_pipeline_kwargs"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document], batch_size: int = 1):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document], batch_size: int = 1):
|
||||
"""
|
||||
Classifies the documents based on the provided labels and adds them to their metadata.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -45,7 +45,7 @@ class OpenAPIConnector:
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, openapi_spec: str, credentials: Optional[Secret] = None, service_kwargs: Optional[Dict[str, Any]] = None
|
||||
self, openapi_spec: str, credentials: Optional[Secret] = None, service_kwargs: Optional[dict[str, Any]] = None
|
||||
):
|
||||
"""
|
||||
Initialize the OpenAPIConnector with a specification and optional credentials.
|
||||
@@ -66,7 +66,7 @@ class OpenAPIConnector:
|
||||
**self.service_kwargs,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
"""
|
||||
@@ -78,15 +78,15 @@ class OpenAPIConnector:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAPIConnector":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAPIConnector":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
"""
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["credentials"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(response=Dict[str, Any])
|
||||
def run(self, operation_id: str, arguments: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
@component.output_types(response=dict[str, Any])
|
||||
def run(self, operation_id: str, arguments: Optional[dict[str, Any]] = None) -> dict[str, Any]:
|
||||
"""
|
||||
Invokes a REST endpoint specified in the OpenAPI specification.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.dataclasses import ChatMessage, ChatRole
|
||||
@@ -25,9 +25,9 @@ with LazyImport("Run 'pip install openapi3'") as openapi_imports:
|
||||
base_url: str,
|
||||
*,
|
||||
data: Optional[Any] = None,
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
parameters: Optional[dict[str, Any]] = None,
|
||||
raw_response: bool = False,
|
||||
security: Optional[Dict[str, str]] = None,
|
||||
security: Optional[dict[str, str]] = None,
|
||||
session: Optional[Any] = None,
|
||||
verify: Union[bool, str] = True,
|
||||
) -> Optional[Any]:
|
||||
@@ -206,13 +206,13 @@ class OpenAPIServiceConnector:
|
||||
openapi_imports.check()
|
||||
self.ssl_verify = ssl_verify
|
||||
|
||||
@component.output_types(service_response=Dict[str, Any])
|
||||
@component.output_types(service_response=dict[str, Any])
|
||||
def run(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
service_openapi_spec: Dict[str, Any],
|
||||
messages: list[ChatMessage],
|
||||
service_openapi_spec: dict[str, Any],
|
||||
service_credentials: Optional[Union[dict, str]] = None,
|
||||
) -> Dict[str, List[ChatMessage]]:
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
"""
|
||||
Processes a list of chat messages to invoke a method on an OpenAPI service.
|
||||
|
||||
@@ -261,7 +261,7 @@ class OpenAPIServiceConnector:
|
||||
|
||||
return {"service_response": response_messages}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -271,7 +271,7 @@ class OpenAPIServiceConnector:
|
||||
return default_to_dict(self, ssl_verify=self.ssl_verify)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAPIServiceConnector":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAPIServiceConnector":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -337,7 +337,7 @@ class OpenAPIServiceConnector:
|
||||
f"for it. Check the service configuration and credentials."
|
||||
)
|
||||
|
||||
def _invoke_method(self, openapi_service: "OpenAPI", method_invocation_descriptor: Dict[str, Any]) -> Any:
|
||||
def _invoke_method(self, openapi_service: "OpenAPI", method_invocation_descriptor: dict[str, Any]) -> Any:
|
||||
"""
|
||||
Invokes the specified method on the OpenAPI service.
|
||||
|
||||
@@ -368,7 +368,7 @@ class OpenAPIServiceConnector:
|
||||
operation_dict = operation.raw_element
|
||||
|
||||
# Pack URL/query parameters under "parameters" key
|
||||
method_call_params: Dict[str, Dict[str, Any]] = defaultdict(dict)
|
||||
method_call_params: dict[str, dict[str, Any]] = defaultdict(dict)
|
||||
parameters = operation_dict.get("parameters", [])
|
||||
request_body = operation_dict.get("requestBody", {})
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import copy
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
import networkx as nx
|
||||
|
||||
@@ -108,8 +108,8 @@ class AzureOCRDocumentConverter:
|
||||
if self.page_layout == "single_column" and self.threshold_y is None:
|
||||
self.threshold_y = 0.05
|
||||
|
||||
@component.output_types(documents=List[Document], raw_azure_response=List[Dict])
|
||||
def run(self, sources: List[Union[str, Path, ByteStream]], meta: Optional[List[Dict[str, Any]]] = None):
|
||||
@component.output_types(documents=list[Document], raw_azure_response=list[dict])
|
||||
def run(self, sources: list[Union[str, Path, ByteStream]], meta: Optional[list[dict[str, Any]]] = None):
|
||||
"""
|
||||
Convert a list of files to Documents using Azure's Document Intelligence service.
|
||||
|
||||
@@ -129,7 +129,7 @@ class AzureOCRDocumentConverter:
|
||||
"""
|
||||
documents = []
|
||||
azure_output = []
|
||||
meta_list: List[Dict[str, Any]] = normalize_metadata(meta=meta, sources_count=len(sources))
|
||||
meta_list: list[dict[str, Any]] = normalize_metadata(meta=meta, sources_count=len(sources))
|
||||
for source, metadata in zip(sources, meta_list):
|
||||
try:
|
||||
bytestream = get_bytestream_from_source(source=source)
|
||||
@@ -152,7 +152,7 @@ class AzureOCRDocumentConverter:
|
||||
|
||||
return {"documents": documents, "raw_azure_response": azure_output}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -173,7 +173,7 @@ class AzureOCRDocumentConverter:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AzureOCRDocumentConverter":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AzureOCRDocumentConverter":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -186,7 +186,7 @@ class AzureOCRDocumentConverter:
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
# pylint: disable=line-too-long
|
||||
def _convert_tables_and_text(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> List[Document]:
|
||||
def _convert_tables_and_text(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> list[Document]:
|
||||
"""
|
||||
Converts the tables and text extracted by Azure's Document Intelligence service into Haystack Documents.
|
||||
|
||||
@@ -205,7 +205,7 @@ class AzureOCRDocumentConverter:
|
||||
docs = [*tables, text]
|
||||
return docs
|
||||
|
||||
def _convert_tables(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> List[Document]:
|
||||
def _convert_tables(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> list[Document]:
|
||||
"""
|
||||
Converts the tables extracted by Azure's Document Intelligence service into Haystack Documents.
|
||||
|
||||
@@ -214,7 +214,7 @@ class AzureOCRDocumentConverter:
|
||||
|
||||
:returns: List of Documents containing the tables extracted from the AnalyzeResult object.
|
||||
"""
|
||||
converted_tables: List[Document] = []
|
||||
converted_tables: list[Document] = []
|
||||
|
||||
if not result.tables:
|
||||
return converted_tables
|
||||
@@ -312,7 +312,7 @@ class AzureOCRDocumentConverter:
|
||||
|
||||
return converted_tables
|
||||
|
||||
def _convert_to_natural_text(self, result: "AnalyzeResult", meta: Optional[Dict[str, Any]]) -> Document:
|
||||
def _convert_to_natural_text(self, result: "AnalyzeResult", meta: Optional[dict[str, Any]]) -> Document:
|
||||
"""
|
||||
This converts the `AnalyzeResult` object into a single document.
|
||||
|
||||
@@ -329,7 +329,7 @@ class AzureOCRDocumentConverter:
|
||||
|
||||
texts = []
|
||||
if result.paragraphs:
|
||||
paragraphs_to_pages: Dict[int, str] = defaultdict(str)
|
||||
paragraphs_to_pages: dict[int, str] = defaultdict(str)
|
||||
for paragraph in result.paragraphs:
|
||||
if paragraph.bounding_regions:
|
||||
# If paragraph spans multiple pages we group it with the first page number
|
||||
@@ -357,7 +357,7 @@ class AzureOCRDocumentConverter:
|
||||
return Document(content=all_text, meta=meta if meta else {})
|
||||
|
||||
def _convert_to_single_column_text(
|
||||
self, result: "AnalyzeResult", meta: Optional[Dict[str, str]], threshold_y: float = 0.05
|
||||
self, result: "AnalyzeResult", meta: Optional[dict[str, str]], threshold_y: float = 0.05
|
||||
) -> Document:
|
||||
"""
|
||||
This converts the `AnalyzeResult` object into a single Haystack Document.
|
||||
@@ -446,7 +446,7 @@ class AzureOCRDocumentConverter:
|
||||
all_text = "\f".join(texts)
|
||||
return Document(content=all_text, meta=meta if meta else {})
|
||||
|
||||
def _collect_table_spans(self, result: "AnalyzeResult") -> Dict:
|
||||
def _collect_table_spans(self, result: "AnalyzeResult") -> dict:
|
||||
"""
|
||||
Collect the spans of all tables by page number.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -50,11 +50,11 @@ class CSVToDocument:
|
||||
self.encoding = encoding
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts a CSV file to a Document.
|
||||
|
||||
@@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass
|
||||
from enum import Enum
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -160,7 +160,7 @@ class DOCXToDocument:
|
||||
self.link_format = DOCXLinkFormat.from_str(link_format) if isinstance(link_format, str) else link_format
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -175,7 +175,7 @@ class DOCXToDocument:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DOCXToDocument":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DOCXToDocument":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -190,11 +190,11 @@ class DOCXToDocument:
|
||||
data["init_parameters"]["link_format"] = DOCXLinkFormat.from_str(data["init_parameters"]["link_format"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts DOCX files to Documents.
|
||||
@@ -246,7 +246,7 @@ class DOCXToDocument:
|
||||
|
||||
return {"documents": documents}
|
||||
|
||||
def _extract_elements(self, document: "DocxDocument") -> List[str]:
|
||||
def _extract_elements(self, document: "DocxDocument") -> list[str]:
|
||||
"""
|
||||
Extracts elements from a DOCX file.
|
||||
|
||||
@@ -331,8 +331,8 @@ class DOCXToDocument:
|
||||
:param table: The DOCX table to convert.
|
||||
:returns: A Markdown string representation of the table.
|
||||
"""
|
||||
markdown: List[str] = []
|
||||
max_col_widths: List[int] = []
|
||||
markdown: list[str] = []
|
||||
max_col_widths: list[int] = []
|
||||
|
||||
# Calculate max width for each column
|
||||
for row in table.rows:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -34,7 +34,7 @@ class HTMLToDocument:
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self, extraction_kwargs: Optional[Dict[str, Any]] = None, store_full_path: bool = False):
|
||||
def __init__(self, extraction_kwargs: Optional[dict[str, Any]] = None, store_full_path: bool = False):
|
||||
"""
|
||||
Create an HTMLToDocument component.
|
||||
|
||||
@@ -50,7 +50,7 @@ class HTMLToDocument:
|
||||
self.extraction_kwargs = extraction_kwargs or {}
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -60,7 +60,7 @@ class HTMLToDocument:
|
||||
return default_to_dict(self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HTMLToDocument":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HTMLToDocument":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -71,12 +71,12 @@ class HTMLToDocument:
|
||||
"""
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
extraction_kwargs: Optional[Dict[str, Any]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
extraction_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Converts a list of HTML files to Documents.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Dict, List, Literal, Optional, Tuple
|
||||
from typing import Literal, Optional
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.image.image_utils import (
|
||||
@@ -68,7 +68,7 @@ class DocumentToImageContent:
|
||||
file_path_meta_field: str = "file_path",
|
||||
root_path: Optional[str] = None,
|
||||
detail: Optional[Literal["auto", "high", "low"]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the DocumentToImageContent component.
|
||||
@@ -90,8 +90,8 @@ class DocumentToImageContent:
|
||||
self.detail = detail
|
||||
self.size = size
|
||||
|
||||
@component.output_types(image_contents=List[Optional[ImageContent]])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Optional[ImageContent]]]:
|
||||
@component.output_types(image_contents=list[Optional[ImageContent]])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Optional[ImageContent]]]:
|
||||
"""
|
||||
Convert documents with image or PDF sources into ImageContent objects.
|
||||
|
||||
@@ -117,9 +117,9 @@ class DocumentToImageContent:
|
||||
documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
|
||||
)
|
||||
|
||||
image_contents: List[Optional[ImageContent]] = [None] * len(documents)
|
||||
image_contents: list[Optional[ImageContent]] = [None] * len(documents)
|
||||
|
||||
pdf_page_infos: List[_PDFPageInfo] = []
|
||||
pdf_page_infos: list[_PDFPageInfo] = []
|
||||
|
||||
for doc_idx, image_source_info in enumerate(images_source_info):
|
||||
mime_type = image_source_info["mime_type"]
|
||||
@@ -142,7 +142,7 @@ class DocumentToImageContent:
|
||||
)
|
||||
|
||||
# efficiently convert PDF pages to images: each PDF is opened and processed only once
|
||||
pdf_page_infos_by_doc_idx: Dict[int, _PDFPageInfo] = {
|
||||
pdf_page_infos_by_doc_idx: dict[int, _PDFPageInfo] = {
|
||||
pdf_page_info["doc_idx"]: pdf_page_info for pdf_page_info in pdf_page_infos
|
||||
}
|
||||
pdf_images_by_doc_idx = _batch_convert_pdf_pages_to_images(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -52,13 +52,13 @@ class ImageFileToDocument:
|
||||
"""
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Convert image files into empty Document objects with metadata.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import component, logging
|
||||
from haystack.components.converters.image.image_utils import _encode_image_to_base64
|
||||
@@ -51,7 +51,7 @@ class ImageFileToImageContent:
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, *, detail: Optional[Literal["auto", "high", "low"]] = None, size: Optional[Tuple[int, int]] = None
|
||||
self, *, detail: Optional[Literal["auto", "high", "low"]] = None, size: Optional[tuple[int, int]] = None
|
||||
):
|
||||
"""
|
||||
Create the ImageFileToImageContent component.
|
||||
@@ -68,15 +68,15 @@ class ImageFileToImageContent:
|
||||
if self.size is not None:
|
||||
pillow_import.check()
|
||||
|
||||
@component.output_types(image_contents=List[ImageContent])
|
||||
@component.output_types(image_contents=list[ImageContent])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
*,
|
||||
detail: Optional[Literal["auto", "high", "low"]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
) -> Dict[str, List[ImageContent]]:
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
) -> dict[str, list[ImageContent]]:
|
||||
"""
|
||||
Converts files to ImageContent objects.
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import mimetypes
|
||||
from collections import defaultdict
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, TypedDict, Union
|
||||
from typing import Optional, TypedDict, Union
|
||||
|
||||
from typing_extensions import NotRequired
|
||||
|
||||
@@ -29,8 +29,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _encode_image_to_base64(
|
||||
bytestream: ByteStream, size: Optional[Tuple[int, int]] = None
|
||||
) -> Tuple[Optional[str], str]:
|
||||
bytestream: ByteStream, size: Optional[tuple[int, int]] = None
|
||||
) -> tuple[Optional[str], str]:
|
||||
"""
|
||||
Encode an image from a ByteStream into a base64-encoded string.
|
||||
|
||||
@@ -118,9 +118,9 @@ def _convert_pdf_to_images(
|
||||
*,
|
||||
bytestream: ByteStream,
|
||||
return_base64: bool = False,
|
||||
page_range: Optional[List[int]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
) -> Union[List[Tuple[int, "Image"]], List[Tuple[int, str]]]:
|
||||
page_range: Optional[list[int]] = None,
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
) -> Union[list[tuple[int, "Image"]], list[tuple[int, str]]]:
|
||||
"""
|
||||
Convert a PDF file into a list of PIL Image objects or base64-encoded images.
|
||||
|
||||
@@ -225,8 +225,8 @@ class _ImageSourceInfo(TypedDict):
|
||||
|
||||
|
||||
def _extract_image_sources_info(
|
||||
documents: List[Document], file_path_meta_field: str, root_path: str
|
||||
) -> List[_ImageSourceInfo]:
|
||||
documents: list[Document], file_path_meta_field: str, root_path: str
|
||||
) -> list[_ImageSourceInfo]:
|
||||
"""
|
||||
Extracts the image source information from the documents.
|
||||
|
||||
@@ -240,7 +240,7 @@ def _extract_image_sources_info(
|
||||
:raises ValueError: If the document is missing the file_path_meta_field key in its metadata, the file path is
|
||||
invalid, the MIME type is not supported, or the page number is missing for a PDF document.
|
||||
"""
|
||||
images_source_info: List[_ImageSourceInfo] = []
|
||||
images_source_info: list[_ImageSourceInfo] = []
|
||||
for doc in documents:
|
||||
file_path = doc.meta.get(file_path_meta_field)
|
||||
if file_path is None:
|
||||
@@ -289,8 +289,8 @@ class _PDFPageInfo(TypedDict):
|
||||
|
||||
|
||||
def _batch_convert_pdf_pages_to_images(
|
||||
*, pdf_page_infos: List[_PDFPageInfo], return_base64: bool = False, size: Optional[Tuple[int, int]] = None
|
||||
) -> Union[Dict[int, str], Dict[int, "Image"]]:
|
||||
*, pdf_page_infos: list[_PDFPageInfo], return_base64: bool = False, size: Optional[tuple[int, int]] = None
|
||||
) -> Union[dict[int, str], dict[int, "Image"]]:
|
||||
"""
|
||||
Converts selected PDF pages to images, returning a mapping from document indices to images (PIL or base64).
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import component, logging
|
||||
from haystack.components.converters.image.image_utils import _convert_pdf_to_images, pillow_import, pypdfium2_import
|
||||
@@ -43,8 +43,8 @@ class PDFToImageContent:
|
||||
self,
|
||||
*,
|
||||
detail: Optional[Literal["auto", "high", "low"]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
page_range: Optional[List[Union[str, int]]] = None,
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
page_range: Optional[list[Union[str, int]]] = None,
|
||||
):
|
||||
"""
|
||||
Create the PDFToImageContent component.
|
||||
@@ -66,16 +66,16 @@ class PDFToImageContent:
|
||||
pypdfium2_import.check()
|
||||
pillow_import.check()
|
||||
|
||||
@component.output_types(image_contents=List[ImageContent])
|
||||
@component.output_types(image_contents=list[ImageContent])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
*,
|
||||
detail: Optional[Literal["auto", "high", "low"]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
page_range: Optional[List[Union[str, int]]] = None,
|
||||
) -> Dict[str, List[ImageContent]]:
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
page_range: Optional[list[Union[str, int]]] = None,
|
||||
) -> dict[str, list[ImageContent]]:
|
||||
"""
|
||||
Converts files to ImageContent objects.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -93,7 +93,7 @@ class JSONConverter:
|
||||
self,
|
||||
jq_schema: Optional[str] = None,
|
||||
content_key: Optional[str] = None,
|
||||
extra_meta_fields: Optional[Union[Set[str], Literal["*"]]] = None,
|
||||
extra_meta_fields: Optional[Union[set[str], Literal["*"]]] = None,
|
||||
store_full_path: bool = False,
|
||||
):
|
||||
"""
|
||||
@@ -149,7 +149,7 @@ class JSONConverter:
|
||||
msg = "No `jq_schema` nor `content_key` specified. Set either or both to extract data."
|
||||
raise ValueError(msg)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -165,7 +165,7 @@ class JSONConverter:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "JSONConverter":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "JSONConverter":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -176,7 +176,7 @@ class JSONConverter:
|
||||
"""
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _get_content_and_meta(self, source: ByteStream) -> List[Tuple[str, Dict[str, Any]]]:
|
||||
def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, Any]]]:
|
||||
"""
|
||||
Utility function to extract text and metadata from a JSON file.
|
||||
|
||||
@@ -246,11 +246,11 @@ class JSONConverter:
|
||||
|
||||
return result
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts a list of JSON files to documents.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
@@ -57,11 +57,11 @@ class MarkdownToDocument:
|
||||
self.progress_bar = progress_bar
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts a list of Markdown files to Documents.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -78,7 +78,7 @@ class MSGToDocument:
|
||||
recip_str += f"{recip.email_address}"
|
||||
return recip_str
|
||||
|
||||
def _convert(self, file_content: io.BytesIO) -> Tuple[str, List[ByteStream]]:
|
||||
def _convert(self, file_content: io.BytesIO) -> tuple[str, list[ByteStream]]:
|
||||
"""
|
||||
Converts the MSG file content into text and extracts any attachments.
|
||||
|
||||
@@ -130,12 +130,12 @@ class MSGToDocument:
|
||||
|
||||
return txt, attachments
|
||||
|
||||
@component.output_types(documents=List[Document], attachments=List[ByteStream])
|
||||
@component.output_types(documents=list[Document], attachments=list[ByteStream])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
) -> Dict[str, Union[List[Document], List[ByteStream]]]:
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
) -> dict[str, Union[list[Document], list[ByteStream]]]:
|
||||
"""
|
||||
Converts MSG files to Documents.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
import yaml
|
||||
|
||||
@@ -52,8 +52,8 @@ class OpenAPIServiceToFunctions:
|
||||
"""
|
||||
openapi_imports.check()
|
||||
|
||||
@component.output_types(functions=List[Dict[str, Any]], openapi_specs=List[Dict[str, Any]])
|
||||
def run(self, sources: List[Union[str, Path, ByteStream]]) -> Dict[str, Any]:
|
||||
@component.output_types(functions=list[dict[str, Any]], openapi_specs=list[dict[str, Any]])
|
||||
def run(self, sources: list[Union[str, Path, ByteStream]]) -> dict[str, Any]:
|
||||
"""
|
||||
Converts OpenAPI definitions in OpenAI function calling format.
|
||||
|
||||
@@ -70,7 +70,7 @@ class OpenAPIServiceToFunctions:
|
||||
:raises ValueError:
|
||||
If the source type is not recognized or no functions are found in the OpenAPI definitions.
|
||||
"""
|
||||
all_extracted_fc_definitions: List[Dict[str, Any]] = []
|
||||
all_extracted_fc_definitions: list[dict[str, Any]] = []
|
||||
all_openapi_specs = []
|
||||
for source in sources:
|
||||
openapi_spec_content = None
|
||||
@@ -101,7 +101,7 @@ class OpenAPIServiceToFunctions:
|
||||
if openapi_spec_content:
|
||||
try:
|
||||
service_openapi_spec = self._parse_openapi_spec(openapi_spec_content)
|
||||
functions: List[Dict[str, Any]] = self._openapi_to_functions(service_openapi_spec)
|
||||
functions: list[dict[str, Any]] = self._openapi_to_functions(service_openapi_spec)
|
||||
all_extracted_fc_definitions.extend(functions)
|
||||
all_openapi_specs.append(service_openapi_spec)
|
||||
except Exception as e:
|
||||
@@ -114,7 +114,7 @@ class OpenAPIServiceToFunctions:
|
||||
|
||||
return {"functions": all_extracted_fc_definitions, "openapi_specs": all_openapi_specs}
|
||||
|
||||
def _openapi_to_functions(self, service_openapi_spec: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
def _openapi_to_functions(self, service_openapi_spec: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""
|
||||
OpenAPI to OpenAI function conversion.
|
||||
|
||||
@@ -122,10 +122,10 @@ class OpenAPIServiceToFunctions:
|
||||
suitable for OpenAI function calling.
|
||||
|
||||
:param service_openapi_spec: The OpenAPI specification from which functions are to be extracted.
|
||||
:type service_openapi_spec: Dict[str, Any]
|
||||
:type service_openapi_spec: dict[str, Any]
|
||||
:return: A list of dictionaries, each representing a function. Each dictionary includes the function's
|
||||
name, description, and a schema of its parameters.
|
||||
:rtype: List[Dict[str, Any]]
|
||||
:rtype: list[dict[str, Any]]
|
||||
"""
|
||||
|
||||
# Doesn't enforce rigid spec validation because that would require a lot of dependencies
|
||||
@@ -142,7 +142,7 @@ class OpenAPIServiceToFunctions:
|
||||
f"at least {OpenAPIServiceToFunctions.MIN_REQUIRED_OPENAPI_SPEC_VERSION}."
|
||||
)
|
||||
|
||||
functions: List[Dict[str, Any]] = []
|
||||
functions: list[dict[str, Any]] = []
|
||||
for paths in service_openapi_spec["paths"].values():
|
||||
for path_spec in paths.values():
|
||||
function_dict = self._parse_endpoint_spec(path_spec)
|
||||
@@ -150,7 +150,7 @@ class OpenAPIServiceToFunctions:
|
||||
functions.append(function_dict)
|
||||
return functions
|
||||
|
||||
def _parse_endpoint_spec(self, resolved_spec: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
def _parse_endpoint_spec(self, resolved_spec: dict[str, Any]) -> Optional[dict[str, Any]]:
|
||||
if not isinstance(resolved_spec, dict):
|
||||
logger.warning("Invalid OpenAPI spec format provided. Could not extract function.")
|
||||
return {}
|
||||
@@ -158,7 +158,7 @@ class OpenAPIServiceToFunctions:
|
||||
function_name = resolved_spec.get("operationId")
|
||||
description = resolved_spec.get("description") or resolved_spec.get("summary", "")
|
||||
|
||||
schema: Dict[str, Any] = {"type": "object", "properties": {}}
|
||||
schema: dict[str, Any] = {"type": "object", "properties": {}}
|
||||
|
||||
# requestBody section
|
||||
req_body_schema = (
|
||||
@@ -191,8 +191,8 @@ class OpenAPIServiceToFunctions:
|
||||
return {}
|
||||
|
||||
def _parse_property_attributes(
|
||||
self, property_schema: Dict[str, Any], include_attributes: Optional[List[str]] = None
|
||||
) -> Dict[str, Any]:
|
||||
self, property_schema: dict[str, Any], include_attributes: Optional[list[str]] = None
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Parses the attributes of a property schema.
|
||||
|
||||
@@ -229,7 +229,7 @@ class OpenAPIServiceToFunctions:
|
||||
|
||||
return parsed_schema
|
||||
|
||||
def _parse_openapi_spec(self, content: str) -> Dict[str, Any]:
|
||||
def _parse_openapi_spec(self, content: str) -> dict[str, Any]:
|
||||
"""
|
||||
Parses OpenAPI specification content, supporting both JSON and YAML formats.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
from typing import Any, Callable, Dict, Optional, Set
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import jinja2.runtime
|
||||
from jinja2 import Environment, TemplateSyntaxError, meta
|
||||
@@ -44,7 +44,7 @@ class OutputAdapter:
|
||||
self,
|
||||
template: str,
|
||||
output_type: TypeAlias,
|
||||
custom_filters: Optional[Dict[str, Callable]] = None,
|
||||
custom_filters: Optional[dict[str, Callable]] = None,
|
||||
unsafe: bool = False,
|
||||
):
|
||||
"""
|
||||
@@ -68,7 +68,7 @@ class OutputAdapter:
|
||||
This should only be used if you trust the source of the template as it can be lead to remote code execution.
|
||||
"""
|
||||
self.custom_filters = {**(custom_filters or {})}
|
||||
input_types: Set[str] = set()
|
||||
input_types: set[str] = set()
|
||||
|
||||
self._unsafe = unsafe
|
||||
|
||||
@@ -137,7 +137,7 @@ class OutputAdapter:
|
||||
raise OutputAdaptationException(f"Error adapting {self.template} with {kwargs}: {e}") from e
|
||||
return adapted_outputs
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -154,7 +154,7 @@ class OutputAdapter:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OutputAdapter":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OutputAdapter":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -174,7 +174,7 @@ class OutputAdapter:
|
||||
}
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _extract_variables(self, env: Environment) -> Set[str]:
|
||||
def _extract_variables(self, env: Environment) -> set[str]:
|
||||
"""
|
||||
Extracts all variables from a list of Jinja template strings.
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import io
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union
|
||||
from typing import Any, Iterator, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -130,7 +130,7 @@ class PDFMinerToDocument:
|
||||
|
||||
return delimited_pages
|
||||
|
||||
def detect_undecoded_cid_characters(self, text: str) -> Dict[str, Any]:
|
||||
def detect_undecoded_cid_characters(self, text: str) -> dict[str, Any]:
|
||||
"""
|
||||
Look for character sequences of CID, i.e.: characters that haven't been properly decoded from their CID format.
|
||||
|
||||
@@ -156,11 +156,11 @@ class PDFMinerToDocument:
|
||||
|
||||
return {"total_chars": total_chars, "cid_chars": cid_chars, "percentage": round(percentage, 2)}
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts PDF files to Documents.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -62,11 +62,11 @@ class PPTXToDocument:
|
||||
text = "\f".join(text_all_slides)
|
||||
return text
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts PPTX files to Documents.
|
||||
|
||||
@@ -6,7 +6,7 @@ import io
|
||||
import os
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -171,11 +171,11 @@ class PyPDFToDocument:
|
||||
text = "\f".join(texts)
|
||||
return text
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts PDF files to documents.
|
||||
|
||||
@@ -6,7 +6,7 @@ import io
|
||||
import os
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -19,7 +19,7 @@ with LazyImport("Run 'pip install tika'") as tika_import:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class XHTMLParser(HTMLParser):
|
||||
class XHTMLParser(HTMLParser): # pylint: disable=abstract-method
|
||||
"""
|
||||
Custom parser to extract pages from Tika XHTML content.
|
||||
"""
|
||||
@@ -28,9 +28,9 @@ class XHTMLParser(HTMLParser):
|
||||
super().__init__()
|
||||
self.ingest = True
|
||||
self.page = ""
|
||||
self.pages: List[str] = []
|
||||
self.pages: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: List[tuple]):
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple]):
|
||||
"""Identify the start of a page div."""
|
||||
if tag == "div" and any(attr == "class" and value == "page" for attr, value in attrs):
|
||||
self.ingest = True
|
||||
@@ -88,11 +88,11 @@ class TikaDocumentConverter:
|
||||
self.tika_url = tika_url
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts files to Documents.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -50,11 +50,11 @@ class TextFileToDocument:
|
||||
self.encoding = encoding
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
):
|
||||
"""
|
||||
Converts text files to documents.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack.dataclasses import ByteStream
|
||||
|
||||
@@ -30,8 +30,8 @@ def get_bytestream_from_source(source: Union[str, Path, ByteStream], guess_mime_
|
||||
|
||||
|
||||
def normalize_metadata(
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]], sources_count: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]], sources_count: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Normalize the metadata input for a converter.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
|
||||
@@ -46,9 +46,9 @@ class XLSXToDocument:
|
||||
def __init__(
|
||||
self,
|
||||
table_format: Literal["csv", "markdown"] = "csv",
|
||||
sheet_name: Union[str, int, List[Union[str, int]], None] = None,
|
||||
read_excel_kwargs: Optional[Dict[str, Any]] = None,
|
||||
table_format_kwargs: Optional[Dict[str, Any]] = None,
|
||||
sheet_name: Union[str, int, list[Union[str, int]], None] = None,
|
||||
read_excel_kwargs: Optional[dict[str, Any]] = None,
|
||||
table_format_kwargs: Optional[dict[str, Any]] = None,
|
||||
*,
|
||||
store_full_path: bool = False,
|
||||
):
|
||||
@@ -79,12 +79,12 @@ class XLSXToDocument:
|
||||
self.table_format_kwargs = table_format_kwargs or {}
|
||||
self.store_full_path = store_full_path
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
sources: List[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
sources: list[Union[str, Path, ByteStream]],
|
||||
meta: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Converts a XLSX file to a Document.
|
||||
|
||||
@@ -136,7 +136,7 @@ class XLSXToDocument:
|
||||
return {"documents": documents}
|
||||
|
||||
@staticmethod
|
||||
def _generate_excel_column_names(n_cols: int) -> List[str]:
|
||||
def _generate_excel_column_names(n_cols: int) -> list[str]:
|
||||
result = []
|
||||
for i in range(n_cols):
|
||||
col_name = ""
|
||||
@@ -147,7 +147,7 @@ class XLSXToDocument:
|
||||
result.append(col_name)
|
||||
return result
|
||||
|
||||
def _extract_tables(self, bytestream: ByteStream) -> Tuple[List[str], List[Dict]]:
|
||||
def _extract_tables(self, bytestream: ByteStream) -> tuple[list[str], list[dict]]:
|
||||
"""
|
||||
Extract tables from a Excel file.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
|
||||
|
||||
@@ -51,14 +51,14 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
|
||||
suffix: str = "",
|
||||
batch_size: int = 32,
|
||||
progress_bar: bool = True,
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
*,
|
||||
default_headers: Optional[Dict[str, str]] = None,
|
||||
default_headers: Optional[dict[str, str]] = None,
|
||||
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
raise_on_failure: bool = False,
|
||||
):
|
||||
"""
|
||||
@@ -146,7 +146,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
|
||||
self.http_client_kwargs = http_client_kwargs
|
||||
self.raise_on_failure = raise_on_failure
|
||||
|
||||
client_args: Dict[str, Any] = {
|
||||
client_args: dict[str, Any] = {
|
||||
"api_version": api_version,
|
||||
"azure_endpoint": azure_endpoint,
|
||||
"azure_deployment": azure_deployment,
|
||||
@@ -166,7 +166,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_args
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -200,7 +200,7 @@ class AzureOpenAIDocumentEmbedder(OpenAIDocumentEmbedder):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIDocumentEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIDocumentEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from openai.lib.azure import AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
|
||||
|
||||
@@ -50,9 +50,9 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
|
||||
prefix: str = "",
|
||||
suffix: str = "",
|
||||
*,
|
||||
default_headers: Optional[Dict[str, str]] = None,
|
||||
default_headers: Optional[dict[str, str]] = None,
|
||||
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an AzureOpenAITextEmbedder component.
|
||||
@@ -127,7 +127,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
|
||||
self.azure_ad_token_provider = azure_ad_token_provider
|
||||
self.http_client_kwargs = http_client_kwargs
|
||||
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
client_kwargs: dict[str, Any] = {
|
||||
"api_version": api_version,
|
||||
"azure_endpoint": azure_endpoint,
|
||||
"azure_deployment": azure_deployment,
|
||||
@@ -147,7 +147,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -176,7 +176,7 @@ class AzureOpenAITextEmbedder(OpenAITextEmbedder):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAITextEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAITextEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack.lazy_imports import LazyImport
|
||||
from haystack.utils.auth import Secret
|
||||
@@ -19,7 +19,7 @@ class _SentenceTransformersEmbeddingBackendFactory:
|
||||
Factory class to create instances of Sentence Transformers embedding backends.
|
||||
"""
|
||||
|
||||
_instances: Dict[str, "_SentenceTransformersEmbeddingBackend"] = {}
|
||||
_instances: dict[str, "_SentenceTransformersEmbeddingBackend"] = {}
|
||||
|
||||
@staticmethod
|
||||
def get_embedding_backend( # pylint: disable=too-many-positional-arguments
|
||||
@@ -29,9 +29,9 @@ class _SentenceTransformersEmbeddingBackendFactory:
|
||||
trust_remote_code: bool = False,
|
||||
local_files_only: bool = False,
|
||||
truncate_dim: Optional[int] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
):
|
||||
embedding_backend_id = f"{model}{device}{auth_token}{truncate_dim}{backend}"
|
||||
@@ -67,9 +67,9 @@ class _SentenceTransformersEmbeddingBackend:
|
||||
trust_remote_code: bool = False,
|
||||
local_files_only: bool = False,
|
||||
truncate_dim: Optional[int] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
):
|
||||
sentence_transformers_import.check()
|
||||
@@ -87,7 +87,7 @@ class _SentenceTransformersEmbeddingBackend:
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
def embed(self, data: Union[List[str], List["Image"]], **kwargs: Any) -> List[List[float]]:
|
||||
def embed(self, data: Union[list[str], list["Image"]], **kwargs: Any) -> list[list[float]]:
|
||||
# Sentence Transformers encode can work with Images, but the type hint does not reflect that
|
||||
# https://sbert.net/examples/sentence_transformer/applications/image-search
|
||||
embeddings = self.model.encode(data, **kwargs).tolist() # type: ignore[arg-type]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from tqdm import tqdm
|
||||
from tqdm.asyncio import tqdm as async_tqdm
|
||||
@@ -92,7 +92,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
def __init__(
|
||||
self,
|
||||
api_type: Union[HFEmbeddingAPIType, str],
|
||||
api_params: Dict[str, str],
|
||||
api_params: dict[str, str],
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
prefix: str = "",
|
||||
suffix: str = "",
|
||||
@@ -100,7 +100,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
normalize: Optional[bool] = False,
|
||||
batch_size: int = 32,
|
||||
progress_bar: bool = True,
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
): # pylint: disable=too-many-positional-arguments
|
||||
"""
|
||||
@@ -168,7 +168,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
msg = f"Unknown api_type {api_type}"
|
||||
raise ValueError(msg)
|
||||
|
||||
client_args: Dict[str, Any] = {"model": model_or_url, "token": token.resolve_value() if token else None}
|
||||
client_args: dict[str, Any] = {"model": model_or_url, "token": token.resolve_value() if token else None}
|
||||
|
||||
self.api_type = api_type
|
||||
self.api_params = api_params
|
||||
@@ -184,7 +184,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
self._client = InferenceClient(**client_args)
|
||||
self._async_client = AsyncInferenceClient(**client_args)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -207,7 +207,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIDocumentEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIDocumentEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -219,7 +219,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["token"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
|
||||
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
||||
"""
|
||||
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
|
||||
"""
|
||||
@@ -239,7 +239,7 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
@staticmethod
|
||||
def _adjust_api_parameters(
|
||||
truncate: Optional[bool], normalize: Optional[bool], api_type: HFEmbeddingAPIType
|
||||
) -> Tuple[Optional[bool], Optional[bool]]:
|
||||
) -> tuple[Optional[bool], Optional[bool]]:
|
||||
"""
|
||||
Adjust the truncate and normalize parameters based on the API type.
|
||||
"""
|
||||
@@ -254,13 +254,13 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
normalize = None
|
||||
return truncate, normalize
|
||||
|
||||
def _embed_batch(self, texts_to_embed: List[str], batch_size: int) -> List[List[float]]:
|
||||
def _embed_batch(self, texts_to_embed: list[str], batch_size: int) -> list[list[float]]:
|
||||
"""
|
||||
Embed a list of texts in batches.
|
||||
"""
|
||||
truncate, normalize = self._adjust_api_parameters(self.truncate, self.normalize, self.api_type)
|
||||
|
||||
all_embeddings: List = []
|
||||
all_embeddings: list = []
|
||||
for i in tqdm(
|
||||
range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
|
||||
):
|
||||
@@ -280,13 +280,13 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
|
||||
return all_embeddings
|
||||
|
||||
async def _embed_batch_async(self, texts_to_embed: List[str], batch_size: int) -> List[List[float]]:
|
||||
async def _embed_batch_async(self, texts_to_embed: list[str], batch_size: int) -> list[list[float]]:
|
||||
"""
|
||||
Embed a list of texts in batches asynchronously.
|
||||
"""
|
||||
truncate, normalize = self._adjust_api_parameters(self.truncate, self.normalize, self.api_type)
|
||||
|
||||
all_embeddings: List = []
|
||||
all_embeddings: list = []
|
||||
for i in async_tqdm(
|
||||
range(0, len(texts_to_embed), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
|
||||
):
|
||||
@@ -306,8 +306,8 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
|
||||
return all_embeddings
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Embeds a list of documents.
|
||||
|
||||
@@ -333,8 +333,8 @@ class HuggingFaceAPIDocumentEmbedder:
|
||||
|
||||
return {"documents": documents}
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
async def run_async(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
async def run_async(self, documents: list[Document]):
|
||||
"""
|
||||
Embeds a list of documents asynchronously.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -75,7 +75,7 @@ class HuggingFaceAPITextEmbedder:
|
||||
def __init__(
|
||||
self,
|
||||
api_type: Union[HFEmbeddingAPIType, str],
|
||||
api_params: Dict[str, str],
|
||||
api_params: dict[str, str],
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
prefix: str = "",
|
||||
suffix: str = "",
|
||||
@@ -171,7 +171,7 @@ class HuggingFaceAPITextEmbedder:
|
||||
|
||||
return text_to_embed, truncate, normalize
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -190,7 +190,7 @@ class HuggingFaceAPITextEmbedder:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPITextEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPITextEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -202,7 +202,7 @@ class HuggingFaceAPITextEmbedder:
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["token"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(embedding=List[float])
|
||||
@component.output_types(embedding=list[float])
|
||||
def run(self, text: str):
|
||||
"""
|
||||
Embeds a single string.
|
||||
@@ -230,7 +230,7 @@ class HuggingFaceAPITextEmbedder:
|
||||
|
||||
return {"embedding": embedding}
|
||||
|
||||
@component.output_types(embedding=List[float])
|
||||
@component.output_types(embedding=list[float])
|
||||
async def run_async(self, text: str):
|
||||
"""
|
||||
Embeds a single string asynchronously.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from copy import copy
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.components.converters.image.image_utils import (
|
||||
@@ -70,11 +70,11 @@ class SentenceTransformersDocumentImageEmbedder:
|
||||
normalize_embeddings: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
local_files_only: bool = False,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
|
||||
encode_kwargs: Optional[Dict[str, Any]] = None,
|
||||
encode_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
) -> None:
|
||||
"""
|
||||
@@ -152,7 +152,7 @@ class SentenceTransformersDocumentImageEmbedder:
|
||||
self.backend = backend
|
||||
self._embedding_backend: Optional[_SentenceTransformersEmbeddingBackend] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -183,7 +183,7 @@ class SentenceTransformersDocumentImageEmbedder:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDocumentImageEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDocumentImageEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -219,8 +219,8 @@ class SentenceTransformersDocumentImageEmbedder:
|
||||
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
|
||||
self._embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Embed a list of documents.
|
||||
|
||||
@@ -243,8 +243,8 @@ class SentenceTransformersDocumentImageEmbedder:
|
||||
documents=documents, file_path_meta_field=self.file_path_meta_field, root_path=self.root_path
|
||||
)
|
||||
|
||||
images_to_embed: List = [None] * len(documents)
|
||||
pdf_page_infos: List[_PDFPageInfo] = []
|
||||
images_to_embed: list = [None] * len(documents)
|
||||
pdf_page_infos: list[_PDFPageInfo] = []
|
||||
|
||||
for doc_idx, image_source_info in enumerate(images_source_info):
|
||||
if image_source_info["mime_type"] == "application/pdf":
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Optional
|
||||
|
||||
from more_itertools import batched
|
||||
from openai import APIError, AsyncOpenAI, OpenAI
|
||||
@@ -50,11 +50,11 @@ class OpenAIDocumentEmbedder:
|
||||
suffix: str = "",
|
||||
batch_size: int = 32,
|
||||
progress_bar: bool = True,
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
*,
|
||||
raise_on_failure: bool = False,
|
||||
):
|
||||
@@ -127,7 +127,7 @@ class OpenAIDocumentEmbedder:
|
||||
if max_retries is None:
|
||||
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
|
||||
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
client_kwargs: dict[str, Any] = {
|
||||
"api_key": api_key.resolve_value(),
|
||||
"organization": organization,
|
||||
"base_url": api_base_url,
|
||||
@@ -140,13 +140,13 @@ class OpenAIDocumentEmbedder:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
|
||||
)
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -173,7 +173,7 @@ class OpenAIDocumentEmbedder:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIDocumentEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAIDocumentEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -185,7 +185,7 @@ class OpenAIDocumentEmbedder:
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _prepare_texts_to_embed(self, documents: List[Document]) -> Dict[str, str]:
|
||||
def _prepare_texts_to_embed(self, documents: list[Document]) -> dict[str, str]:
|
||||
"""
|
||||
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
|
||||
"""
|
||||
@@ -202,18 +202,18 @@ class OpenAIDocumentEmbedder:
|
||||
return texts_to_embed
|
||||
|
||||
def _embed_batch(
|
||||
self, texts_to_embed: Dict[str, str], batch_size: int
|
||||
) -> Tuple[Dict[str, List[float]], Dict[str, Any]]:
|
||||
self, texts_to_embed: dict[str, str], batch_size: int
|
||||
) -> tuple[dict[str, list[float]], dict[str, Any]]:
|
||||
"""
|
||||
Embed a list of texts in batches.
|
||||
"""
|
||||
|
||||
doc_ids_to_embeddings: Dict[str, List[float]] = {}
|
||||
meta: Dict[str, Any] = {}
|
||||
doc_ids_to_embeddings: dict[str, list[float]] = {}
|
||||
meta: dict[str, Any] = {}
|
||||
for batch in tqdm(
|
||||
batched(texts_to_embed.items(), batch_size), disable=not self.progress_bar, desc="Calculating embeddings"
|
||||
):
|
||||
args: Dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch], "encoding_format": "float"}
|
||||
args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch], "encoding_format": "float"}
|
||||
|
||||
if self.dimensions is not None:
|
||||
args["dimensions"] = self.dimensions
|
||||
@@ -242,21 +242,21 @@ class OpenAIDocumentEmbedder:
|
||||
return doc_ids_to_embeddings, meta
|
||||
|
||||
async def _embed_batch_async(
|
||||
self, texts_to_embed: Dict[str, str], batch_size: int
|
||||
) -> Tuple[Dict[str, List[float]], Dict[str, Any]]:
|
||||
self, texts_to_embed: dict[str, str], batch_size: int
|
||||
) -> tuple[dict[str, list[float]], dict[str, Any]]:
|
||||
"""
|
||||
Embed a list of texts in batches asynchronously.
|
||||
"""
|
||||
|
||||
doc_ids_to_embeddings: Dict[str, List[float]] = {}
|
||||
meta: Dict[str, Any] = {}
|
||||
doc_ids_to_embeddings: dict[str, list[float]] = {}
|
||||
meta: dict[str, Any] = {}
|
||||
|
||||
batches = list(batched(texts_to_embed.items(), batch_size))
|
||||
if self.progress_bar:
|
||||
batches = async_tqdm(batches, desc="Calculating embeddings")
|
||||
|
||||
for batch in batches:
|
||||
args: Dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch]}
|
||||
args: dict[str, Any] = {"model": self.model, "input": [b[1] for b in batch]}
|
||||
|
||||
if self.dimensions is not None:
|
||||
args["dimensions"] = self.dimensions
|
||||
@@ -284,8 +284,8 @@ class OpenAIDocumentEmbedder:
|
||||
|
||||
return doc_ids_to_embeddings, meta
|
||||
|
||||
@component.output_types(documents=List[Document], meta=Dict[str, Any])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document], meta=dict[str, Any])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Embeds a list of documents.
|
||||
|
||||
@@ -313,8 +313,8 @@ class OpenAIDocumentEmbedder:
|
||||
|
||||
return {"documents": list(doc_id_to_document.values()), "meta": meta}
|
||||
|
||||
@component.output_types(documents=List[Document], meta=Dict[str, Any])
|
||||
async def run_async(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document], meta=dict[str, Any])
|
||||
async def run_async(self, documents: list[Document]):
|
||||
"""
|
||||
Embeds a list of documents asynchronously.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
@@ -48,7 +48,7 @@ class OpenAITextEmbedder:
|
||||
suffix: str = "",
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an OpenAITextEmbedder component.
|
||||
@@ -103,7 +103,7 @@ class OpenAITextEmbedder:
|
||||
if max_retries is None:
|
||||
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
|
||||
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
client_kwargs: dict[str, Any] = {
|
||||
"api_key": api_key.resolve_value(),
|
||||
"organization": organization,
|
||||
"base_url": api_base_url,
|
||||
@@ -116,13 +116,13 @@ class OpenAITextEmbedder:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
|
||||
)
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -144,7 +144,7 @@ class OpenAITextEmbedder:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAITextEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAITextEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -156,7 +156,7 @@ class OpenAITextEmbedder:
|
||||
deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _prepare_input(self, text: str) -> Dict[str, Any]:
|
||||
def _prepare_input(self, text: str) -> dict[str, Any]:
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(
|
||||
"OpenAITextEmbedder expects a string as an input."
|
||||
@@ -165,15 +165,15 @@ class OpenAITextEmbedder:
|
||||
|
||||
text_to_embed = self.prefix + text + self.suffix
|
||||
|
||||
kwargs: Dict[str, Any] = {"model": self.model, "input": text_to_embed, "encoding_format": "float"}
|
||||
kwargs: dict[str, Any] = {"model": self.model, "input": text_to_embed, "encoding_format": "float"}
|
||||
if self.dimensions is not None:
|
||||
kwargs["dimensions"] = self.dimensions
|
||||
return kwargs
|
||||
|
||||
def _prepare_output(self, result: CreateEmbeddingResponse) -> Dict[str, Any]:
|
||||
def _prepare_output(self, result: CreateEmbeddingResponse) -> dict[str, Any]:
|
||||
return {"embedding": result.data[0].embedding, "meta": {"model": result.model, "usage": dict(result.usage)}}
|
||||
|
||||
@component.output_types(embedding=List[float], meta=Dict[str, Any])
|
||||
@component.output_types(embedding=list[float], meta=dict[str, Any])
|
||||
def run(self, text: str):
|
||||
"""
|
||||
Embeds a single string.
|
||||
@@ -190,7 +190,7 @@ class OpenAITextEmbedder:
|
||||
response = self.client.embeddings.create(**create_kwargs)
|
||||
return self._prepare_output(result=response)
|
||||
|
||||
@component.output_types(embedding=List[float], meta=Dict[str, Any])
|
||||
@component.output_types(embedding=list[float], meta=dict[str, Any])
|
||||
async def run_async(self, text: str):
|
||||
"""
|
||||
Asynchronously embed a single string.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.components.embedders.backends.sentence_transformers_backend import (
|
||||
@@ -49,16 +49,16 @@ class SentenceTransformersDocumentEmbedder:
|
||||
batch_size: int = 32,
|
||||
progress_bar: bool = True,
|
||||
normalize_embeddings: bool = False,
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
trust_remote_code: bool = False,
|
||||
local_files_only: bool = False,
|
||||
truncate_dim: Optional[int] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
|
||||
encode_kwargs: Optional[Dict[str, Any]] = None,
|
||||
encode_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
):
|
||||
"""
|
||||
@@ -141,13 +141,13 @@ class SentenceTransformersDocumentEmbedder:
|
||||
self.precision = precision
|
||||
self.backend = backend
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -181,7 +181,7 @@ class SentenceTransformersDocumentEmbedder:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDocumentEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDocumentEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -218,8 +218,8 @@ class SentenceTransformersDocumentEmbedder:
|
||||
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
|
||||
self.embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Embed a list of documents.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.components.embedders.backends.sentence_transformers_backend import (
|
||||
@@ -48,11 +48,11 @@ class SentenceTransformersTextEmbedder:
|
||||
trust_remote_code: bool = False,
|
||||
local_files_only: bool = False,
|
||||
truncate_dim: Optional[int] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
precision: Literal["float32", "int8", "uint8", "binary", "ubinary"] = "float32",
|
||||
encode_kwargs: Optional[Dict[str, Any]] = None,
|
||||
encode_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
):
|
||||
"""
|
||||
@@ -129,13 +129,13 @@ class SentenceTransformersTextEmbedder:
|
||||
self.precision = precision
|
||||
self.backend = backend
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -167,7 +167,7 @@ class SentenceTransformersTextEmbedder:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersTextEmbedder":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersTextEmbedder":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -204,7 +204,7 @@ class SentenceTransformersTextEmbedder:
|
||||
if self.tokenizer_kwargs and self.tokenizer_kwargs.get("model_max_length"):
|
||||
self.embedding_backend.model.max_seq_length = self.tokenizer_kwargs["model_max_length"]
|
||||
|
||||
@component.output_types(embedding=List[float])
|
||||
@component.output_types(embedding=list[float])
|
||||
def run(self, text: str):
|
||||
"""
|
||||
Embed a single string.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, Protocol
|
||||
from typing import Any, Protocol
|
||||
|
||||
# See https://github.com/pylint-dev/pylint/issues/9319.
|
||||
# pylint: disable=unnecessary-ellipsis
|
||||
@@ -13,7 +13,7 @@ class TextEmbedder(Protocol):
|
||||
Protocol for Text Embedders.
|
||||
"""
|
||||
|
||||
def run(self, text: str) -> Dict[str, Any]:
|
||||
def run(self, text: str) -> dict[str, Any]:
|
||||
"""
|
||||
Generate embeddings for the input text.
|
||||
|
||||
@@ -24,7 +24,7 @@ class TextEmbedder(Protocol):
|
||||
The input text to be embedded.
|
||||
:returns:
|
||||
A dictionary containing the keys:
|
||||
- 'embedding', which is expected to be a List[float] representing the embedding.
|
||||
- 'embedding', which is expected to be a list[float] representing the embedding.
|
||||
- any optional keys such as 'metadata'.
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
@@ -35,8 +35,8 @@ class AnswerExactMatchEvaluator:
|
||||
```
|
||||
"""
|
||||
|
||||
@component.output_types(individual_scores=List[int], score=float)
|
||||
def run(self, ground_truth_answers: List[str], predicted_answers: List[str]) -> Dict[str, Any]:
|
||||
@component.output_types(individual_scores=list[int], score=float)
|
||||
def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
|
||||
"""
|
||||
Run the AnswerExactMatchEvaluator on the given inputs.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from statistics import mean
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.components.evaluators.llm_evaluator import LLMEvaluator
|
||||
@@ -99,7 +99,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
|
||||
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
examples: Optional[List[Dict[str, Any]]] = None,
|
||||
examples: Optional[list[dict[str, Any]]] = None,
|
||||
progress_bar: bool = True,
|
||||
raise_on_failure: bool = True,
|
||||
chat_generator: Optional[ChatGenerator] = None,
|
||||
@@ -142,7 +142,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
|
||||
"required to answer the following question. If no relevant sentences are found, or if you "
|
||||
"believe the question cannot be answered from the given context, return an empty list, example: []"
|
||||
)
|
||||
self.inputs = [("questions", List[str]), ("contexts", List[List[str]])]
|
||||
self.inputs = [("questions", list[str]), ("contexts", list[list[str]])]
|
||||
self.outputs = ["relevant_statements"]
|
||||
self.examples = examples or _DEFAULT_EXAMPLES
|
||||
|
||||
@@ -156,8 +156,8 @@ class ContextRelevanceEvaluator(LLMEvaluator):
|
||||
progress_bar=progress_bar,
|
||||
)
|
||||
|
||||
@component.output_types(score=float, results=List[Dict[str, Any]])
|
||||
def run(self, **inputs) -> Dict[str, Any]:
|
||||
@component.output_types(score=float, results=list[dict[str, Any]])
|
||||
def run(self, **inputs) -> dict[str, Any]:
|
||||
"""
|
||||
Run the LLM evaluator.
|
||||
|
||||
@@ -187,7 +187,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
|
||||
|
||||
return result
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -203,7 +203,7 @@ class ContextRelevanceEvaluator(LLMEvaluator):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ContextRelevanceEvaluator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "ContextRelevanceEvaluator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from haystack import Document, component
|
||||
|
||||
@@ -44,10 +44,10 @@ class DocumentMAPEvaluator:
|
||||
"""
|
||||
|
||||
# Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
|
||||
@component.output_types(score=float, individual_scores=List[float])
|
||||
@component.output_types(score=float, individual_scores=list[float])
|
||||
def run(
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentMAPEvaluator on the given inputs.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from haystack import Document, component
|
||||
|
||||
@@ -42,10 +42,10 @@ class DocumentMRREvaluator:
|
||||
"""
|
||||
|
||||
# Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
|
||||
@component.output_types(score=float, individual_scores=List[float])
|
||||
@component.output_types(score=float, individual_scores=list[float])
|
||||
def run(
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentMRREvaluator on the given inputs.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from math import log2
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from haystack import Document, component
|
||||
|
||||
@@ -34,10 +34,10 @@ class DocumentNDCGEvaluator:
|
||||
```
|
||||
"""
|
||||
|
||||
@component.output_types(score=float, individual_scores=List[float])
|
||||
@component.output_types(score=float, individual_scores=list[float])
|
||||
def run(
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentNDCGEvaluator on the given inputs.
|
||||
|
||||
@@ -68,7 +68,7 @@ class DocumentNDCGEvaluator:
|
||||
return {"score": score, "individual_scores": individual_scores}
|
||||
|
||||
@staticmethod
|
||||
def validate_inputs(gt_docs: List[List[Document]], ret_docs: List[List[Document]]):
|
||||
def validate_inputs(gt_docs: list[list[Document]], ret_docs: list[list[Document]]):
|
||||
"""
|
||||
Validate the input parameters.
|
||||
|
||||
@@ -96,7 +96,7 @@ class DocumentNDCGEvaluator:
|
||||
raise ValueError(msg)
|
||||
|
||||
@staticmethod
|
||||
def calculate_dcg(gt_docs: List[Document], ret_docs: List[Document]) -> float:
|
||||
def calculate_dcg(gt_docs: list[Document], ret_docs: list[Document]) -> float:
|
||||
"""
|
||||
Calculate the discounted cumulative gain (DCG) of the retrieved documents.
|
||||
|
||||
@@ -116,7 +116,7 @@ class DocumentNDCGEvaluator:
|
||||
return dcg
|
||||
|
||||
@staticmethod
|
||||
def calculate_idcg(gt_docs: List[Document]) -> float:
|
||||
def calculate_idcg(gt_docs: list[Document]) -> float:
|
||||
"""
|
||||
Calculate the ideal discounted cumulative gain (IDCG) of the ground truth documents.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Union
|
||||
from typing import Any, Union
|
||||
|
||||
from haystack import component, default_to_dict, logging
|
||||
from haystack.dataclasses import Document
|
||||
@@ -86,7 +86,7 @@ class DocumentRecallEvaluator:
|
||||
self.mode = mode
|
||||
|
||||
@staticmethod
|
||||
def _recall_single_hit(ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> float:
|
||||
def _recall_single_hit(ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
|
||||
unique_truths = {g.content for g in ground_truth_documents}
|
||||
unique_retrievals = {p.content for p in retrieved_documents}
|
||||
retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
|
||||
@@ -94,7 +94,7 @@ class DocumentRecallEvaluator:
|
||||
return float(len(retrieved_ground_truths) > 0)
|
||||
|
||||
@staticmethod
|
||||
def _recall_multi_hit(ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> float:
|
||||
def _recall_multi_hit(ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
|
||||
unique_truths = {g.content for g in ground_truth_documents}
|
||||
unique_retrievals = {p.content for p in retrieved_documents}
|
||||
retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
|
||||
@@ -115,10 +115,10 @@ class DocumentRecallEvaluator:
|
||||
|
||||
return len(retrieved_ground_truths) / len(unique_truths)
|
||||
|
||||
@component.output_types(score=float, individual_scores=List[float])
|
||||
@component.output_types(score=float, individual_scores=list[float])
|
||||
def run(
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentRecallEvaluator on the given inputs.
|
||||
|
||||
@@ -144,7 +144,7 @@ class DocumentRecallEvaluator:
|
||||
|
||||
return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from numpy import mean as np_mean
|
||||
|
||||
@@ -86,7 +86,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
|
||||
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
examples: Optional[List[Dict[str, Any]]] = None,
|
||||
examples: Optional[list[dict[str, Any]]] = None,
|
||||
progress_bar: bool = True,
|
||||
raise_on_failure: bool = True,
|
||||
chat_generator: Optional[ChatGenerator] = None,
|
||||
@@ -132,7 +132,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
|
||||
"statement made in the predicted answer. The score is 1 if the statement can be "
|
||||
"inferred from the provided context or 0 if it cannot be inferred."
|
||||
)
|
||||
self.inputs = [("questions", List[str]), ("contexts", List[List[str]]), ("predicted_answers", List[str])]
|
||||
self.inputs = [("questions", list[str]), ("contexts", list[list[str]]), ("predicted_answers", list[str])]
|
||||
self.outputs = ["statements", "statement_scores"]
|
||||
self.examples = examples or _DEFAULT_EXAMPLES
|
||||
|
||||
@@ -146,8 +146,8 @@ class FaithfulnessEvaluator(LLMEvaluator):
|
||||
progress_bar=progress_bar,
|
||||
)
|
||||
|
||||
@component.output_types(individual_scores=List[int], score=float, results=List[Dict[str, Any]])
|
||||
def run(self, **inputs) -> Dict[str, Any]:
|
||||
@component.output_types(individual_scores=list[int], score=float, results=list[dict[str, Any]])
|
||||
def run(self, **inputs) -> dict[str, Any]:
|
||||
"""
|
||||
Run the LLM evaluator.
|
||||
|
||||
@@ -181,7 +181,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
|
||||
|
||||
return result
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -197,7 +197,7 @@ class FaithfulnessEvaluator(LLMEvaluator):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "FaithfulnessEvaluator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "FaithfulnessEvaluator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
from typing import Any, Optional
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
@@ -30,11 +30,10 @@ class LLMEvaluator:
|
||||
|
||||
Usage example:
|
||||
```python
|
||||
from typing import List
|
||||
from haystack.components.evaluators import LLMEvaluator
|
||||
evaluator = LLMEvaluator(
|
||||
instructions="Is this answer problematic for children?",
|
||||
inputs=[("predicted_answers", List[str])],
|
||||
inputs=[("predicted_answers", list[str])],
|
||||
outputs=["score"],
|
||||
examples=[
|
||||
{"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
|
||||
@@ -54,9 +53,9 @@ class LLMEvaluator:
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
instructions: str,
|
||||
inputs: List[Tuple[str, Type[List]]],
|
||||
outputs: List[str],
|
||||
examples: List[Dict[str, Any]],
|
||||
inputs: list[tuple[str, type[list]]],
|
||||
outputs: list[str],
|
||||
examples: list[dict[str, Any]],
|
||||
progress_bar: bool = True,
|
||||
*,
|
||||
raise_on_failure: bool = True,
|
||||
@@ -111,7 +110,7 @@ class LLMEvaluator:
|
||||
|
||||
@staticmethod
|
||||
def validate_init_parameters(
|
||||
inputs: List[Tuple[str, Type[List]]], outputs: List[str], examples: List[Dict[str, Any]]
|
||||
inputs: list[tuple[str, type[list]]], outputs: list[str], examples: list[dict[str, Any]]
|
||||
):
|
||||
"""
|
||||
Validate the init parameters.
|
||||
@@ -164,8 +163,8 @@ class LLMEvaluator:
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
@component.output_types(results=List[Dict[str, Any]])
|
||||
def run(self, **inputs) -> Dict[str, Any]:
|
||||
@component.output_types(results=list[dict[str, Any]])
|
||||
def run(self, **inputs) -> dict[str, Any]:
|
||||
"""
|
||||
Run the LLM evaluator.
|
||||
|
||||
@@ -189,7 +188,7 @@ class LLMEvaluator:
|
||||
input_names, values = inputs.keys(), list(zip(*inputs.values()))
|
||||
list_of_input_names_to_values = [dict(zip(input_names, v)) for v in values]
|
||||
|
||||
results: List[Optional[Dict[str, Any]]] = []
|
||||
results: list[Optional[dict[str, Any]]] = []
|
||||
metadata = []
|
||||
errors = 0
|
||||
for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
|
||||
@@ -269,7 +268,7 @@ class LLMEvaluator:
|
||||
f"Outputs:\n"
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -289,7 +288,7 @@ class LLMEvaluator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "LLMEvaluator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "LLMEvaluator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
@@ -308,7 +307,7 @@ class LLMEvaluator:
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@staticmethod
|
||||
def validate_input_parameters(expected: Dict[str, Any], received: Dict[str, Any]) -> None:
|
||||
def validate_input_parameters(expected: dict[str, Any], received: dict[str, Any]) -> None:
|
||||
"""
|
||||
Validate the input parameters.
|
||||
|
||||
@@ -345,7 +344,7 @@ class LLMEvaluator:
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
def is_valid_json_and_has_expected_keys(self, expected: List[str], received: str) -> bool:
|
||||
def is_valid_json_and_has_expected_keys(self, expected: list[str], received: str) -> bool:
|
||||
"""
|
||||
Output must be a valid JSON with the expected keys.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from numpy import mean as np_mean
|
||||
|
||||
@@ -82,7 +82,7 @@ class SASEvaluator:
|
||||
self._token = token
|
||||
self._similarity_model: Union[SentenceTransformer, CrossEncoder, None] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -98,7 +98,7 @@ class SASEvaluator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SASEvaluator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SASEvaluator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
@@ -137,8 +137,8 @@ class SASEvaluator:
|
||||
else:
|
||||
self._similarity_model = SentenceTransformer(self._model, device=device, use_auth_token=token)
|
||||
|
||||
@component.output_types(score=float, individual_scores=List[float])
|
||||
def run(self, ground_truth_answers: List[str], predicted_answers: List[str]) -> Dict[str, Any]:
|
||||
@component.output_types(score=float, individual_scores=list[float])
|
||||
def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
|
||||
"""
|
||||
SASEvaluator component run method.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from jinja2 import meta
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
@@ -92,7 +92,7 @@ class LLMDocumentContentExtractor:
|
||||
file_path_meta_field: str = "file_path",
|
||||
root_path: Optional[str] = None,
|
||||
detail: Optional[Literal["auto", "high", "low"]] = None,
|
||||
size: Optional[Tuple[int, int]] = None,
|
||||
size: Optional[tuple[int, int]] = None,
|
||||
raise_on_failure: bool = False,
|
||||
max_workers: int = 3,
|
||||
):
|
||||
@@ -147,7 +147,7 @@ class LLMDocumentContentExtractor:
|
||||
self._chat_generator.warm_up()
|
||||
self._is_warmed_up = True
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -168,7 +168,7 @@ class LLMDocumentContentExtractor:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "LLMDocumentContentExtractor":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "LLMDocumentContentExtractor":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -180,7 +180,7 @@ class LLMDocumentContentExtractor:
|
||||
deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _run_on_thread(self, message: Optional[ChatMessage]) -> Dict[str, Any]:
|
||||
def _run_on_thread(self, message: Optional[ChatMessage]) -> dict[str, Any]:
|
||||
"""
|
||||
Execute the LLM inference in a separate thread for each document.
|
||||
|
||||
@@ -205,8 +205,8 @@ class LLMDocumentContentExtractor:
|
||||
result = {"error": "LLM failed with exception: " + str(e)}
|
||||
return result
|
||||
|
||||
@component.output_types(documents=List[Document], failed_documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
|
||||
@component.output_types(documents=list[Document], failed_documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Run content extraction on a list of image-based documents using a vision-capable LLM.
|
||||
|
||||
@@ -225,7 +225,7 @@ class LLMDocumentContentExtractor:
|
||||
|
||||
# Create ChatMessage prompts for each document
|
||||
image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
|
||||
all_messages: List[Union[ChatMessage, None]] = []
|
||||
all_messages: list[Union[ChatMessage, None]] = []
|
||||
for image_content in image_contents:
|
||||
if image_content is None:
|
||||
# If the image content is None, it means the document could not be converted to an image.
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import copy
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from jinja2 import meta
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
@@ -127,8 +127,8 @@ class LLMMetadataExtractor:
|
||||
self,
|
||||
prompt: str,
|
||||
chat_generator: ChatGenerator,
|
||||
expected_keys: Optional[List[str]] = None,
|
||||
page_range: Optional[List[Union[str, int]]] = None,
|
||||
expected_keys: Optional[list[str]] = None,
|
||||
page_range: Optional[list[Union[str, int]]] = None,
|
||||
raise_on_failure: bool = False,
|
||||
max_workers: int = 3,
|
||||
):
|
||||
@@ -172,7 +172,7 @@ class LLMMetadataExtractor:
|
||||
if hasattr(self._chat_generator, "warm_up"):
|
||||
self._chat_generator.warm_up()
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -191,7 +191,7 @@ class LLMMetadataExtractor:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "LLMMetadataExtractor":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "LLMMetadataExtractor":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -204,8 +204,8 @@ class LLMMetadataExtractor:
|
||||
deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _extract_metadata(self, llm_answer: str) -> Dict[str, Any]:
|
||||
parsed_metadata: Dict[str, Any] = {}
|
||||
def _extract_metadata(self, llm_answer: str) -> dict[str, Any]:
|
||||
parsed_metadata: dict[str, Any] = {}
|
||||
|
||||
try:
|
||||
parsed_metadata = json.loads(llm_answer)
|
||||
@@ -229,9 +229,9 @@ class LLMMetadataExtractor:
|
||||
return parsed_metadata
|
||||
|
||||
def _prepare_prompts(
|
||||
self, documents: List[Document], expanded_range: Optional[List[int]] = None
|
||||
) -> List[Union[ChatMessage, None]]:
|
||||
all_prompts: List[Union[ChatMessage, None]] = []
|
||||
self, documents: list[Document], expanded_range: Optional[list[int]] = None
|
||||
) -> list[Union[ChatMessage, None]]:
|
||||
all_prompts: list[Union[ChatMessage, None]] = []
|
||||
for document in documents:
|
||||
if not document.content:
|
||||
logger.warning("Document {doc_id} has no content. Skipping metadata extraction.", doc_id=document.id)
|
||||
@@ -257,7 +257,7 @@ class LLMMetadataExtractor:
|
||||
|
||||
return all_prompts
|
||||
|
||||
def _run_on_thread(self, prompt: Optional[ChatMessage]) -> Dict[str, Any]:
|
||||
def _run_on_thread(self, prompt: Optional[ChatMessage]) -> dict[str, Any]:
|
||||
# If prompt is None, return an error dictionary
|
||||
if prompt is None:
|
||||
return {"error": "Document has no content, skipping LLM call."}
|
||||
@@ -275,8 +275,8 @@ class LLMMetadataExtractor:
|
||||
result = {"error": "LLM failed with exception: " + str(e)}
|
||||
return result
|
||||
|
||||
@component.output_types(documents=List[Document], failed_documents=List[Document])
|
||||
def run(self, documents: List[Document], page_range: Optional[List[Union[str, int]]] = None):
|
||||
@component.output_types(documents=list[Document], failed_documents=list[Document])
|
||||
def run(self, documents: list[Document], page_range: Optional[list[Union[str, int]]] = None):
|
||||
"""
|
||||
Extract metadata from documents using a Large Language Model.
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import ComponentError, DeserializationError, Document, component, default_from_dict, default_to_dict
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -110,7 +110,7 @@ class NamedEntityExtractor:
|
||||
*,
|
||||
backend: Union[str, NamedEntityExtractorBackend],
|
||||
model: str,
|
||||
pipeline_kwargs: Optional[Dict[str, Any]] = None,
|
||||
pipeline_kwargs: Optional[dict[str, Any]] = None,
|
||||
device: Optional[ComponentDevice] = None,
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
) -> None:
|
||||
@@ -177,8 +177,8 @@ class NamedEntityExtractor:
|
||||
f"Named entity extractor with backend '{self._backend.type}' failed to initialize."
|
||||
) from e
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document], batch_size: int = 1) -> Dict[str, Any]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document], batch_size: int = 1) -> dict[str, Any]:
|
||||
"""
|
||||
Annotate named entities in each document and store the annotations in the document's metadata.
|
||||
|
||||
@@ -209,7 +209,7 @@ class NamedEntityExtractor:
|
||||
|
||||
return {"documents": documents}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -232,7 +232,7 @@ class NamedEntityExtractor:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "NamedEntityExtractor":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "NamedEntityExtractor":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -262,7 +262,7 @@ class NamedEntityExtractor:
|
||||
return self._backend.initialized
|
||||
|
||||
@classmethod
|
||||
def get_stored_annotations(cls, document: Document) -> Optional[List[NamedEntityAnnotation]]:
|
||||
def get_stored_annotations(cls, document: Document) -> Optional[list[NamedEntityAnnotation]]:
|
||||
"""
|
||||
Returns the document's named entity annotations stored in its metadata, if any.
|
||||
|
||||
@@ -284,7 +284,7 @@ class _NerBackend(ABC):
|
||||
self,
|
||||
_type: NamedEntityExtractorBackend,
|
||||
device: ComponentDevice,
|
||||
pipeline_kwargs: Optional[Dict[str, Any]] = None,
|
||||
pipeline_kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -306,7 +306,7 @@ class _NerBackend(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
|
||||
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
|
||||
"""
|
||||
Predict annotations for a collection of documents.
|
||||
|
||||
@@ -350,7 +350,7 @@ class _HfBackend(_NerBackend):
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[Dict[str, Any]] = None
|
||||
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Construct a Hugging Face NER backend.
|
||||
@@ -382,7 +382,7 @@ class _HfBackend(_NerBackend):
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(self._model_name_or_path, token=token)
|
||||
self.model = AutoModelForTokenClassification.from_pretrained(self._model_name_or_path, token=token)
|
||||
|
||||
pipeline_params: Dict[str, Any] = {
|
||||
pipeline_params: dict[str, Any] = {
|
||||
"task": "ner",
|
||||
"model": self.model,
|
||||
"tokenizer": self.tokenizer,
|
||||
@@ -392,7 +392,7 @@ class _HfBackend(_NerBackend):
|
||||
self.device.update_hf_kwargs(pipeline_params, overwrite=False)
|
||||
self.pipeline = pipeline(**pipeline_params)
|
||||
|
||||
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
|
||||
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
|
||||
if not self.initialized:
|
||||
raise ComponentError("Hugging Face NER backend was not initialized - Did you call `warm_up()`?")
|
||||
|
||||
@@ -426,7 +426,7 @@ class _SpacyBackend(_NerBackend):
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[Dict[str, Any]] = None
|
||||
self, *, model_name_or_path: str, device: ComponentDevice, pipeline_kwargs: Optional[dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""
|
||||
Construct a spaCy NER backend.
|
||||
@@ -467,7 +467,7 @@ class _SpacyBackend(_NerBackend):
|
||||
|
||||
self._pipeline_kwargs = {k: v for k, v in self._pipeline_kwargs.items() if k not in ("texts", "batch_size")}
|
||||
|
||||
def annotate(self, texts: List[str], *, batch_size: int = 1) -> List[List[NamedEntityAnnotation]]:
|
||||
def annotate(self, texts: list[str], *, batch_size: int = 1) -> list[list[NamedEntityAnnotation]]:
|
||||
if not self.initialized:
|
||||
raise ComponentError("spaCy NER backend was not initialized - Did you call `warm_up()`?")
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import asyncio
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from fnmatch import fnmatch
|
||||
from typing import Callable, Dict, List, Optional, Tuple, cast
|
||||
from typing import Callable, Optional, cast
|
||||
|
||||
import httpx
|
||||
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
|
||||
@@ -95,11 +95,11 @@ class LinkContentFetcher:
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
raise_on_failure: bool = True,
|
||||
user_agents: Optional[List[str]] = None,
|
||||
user_agents: Optional[list[str]] = None,
|
||||
retry_attempts: int = 2,
|
||||
timeout: int = 3,
|
||||
http2: bool = False,
|
||||
client_kwargs: Optional[Dict] = None,
|
||||
client_kwargs: Optional[dict] = None,
|
||||
):
|
||||
"""
|
||||
Initializes the component.
|
||||
@@ -149,7 +149,7 @@ class LinkContentFetcher:
|
||||
self._async_client = httpx.AsyncClient(**client_kwargs)
|
||||
|
||||
# register default content handlers that extract data from the response
|
||||
self.handlers: Dict[str, Callable[[httpx.Response], ByteStream]] = defaultdict(lambda: _text_content_handler)
|
||||
self.handlers: dict[str, Callable[[httpx.Response], ByteStream]] = defaultdict(lambda: _text_content_handler)
|
||||
self.handlers["text/*"] = _text_content_handler
|
||||
self.handlers["text/html"] = _binary_content_handler
|
||||
self.handlers["application/json"] = _text_content_handler
|
||||
@@ -193,8 +193,8 @@ class LinkContentFetcher:
|
||||
# Suppress any exceptions during cleanup
|
||||
pass
|
||||
|
||||
@component.output_types(streams=List[ByteStream])
|
||||
def run(self, urls: List[str]):
|
||||
@component.output_types(streams=list[ByteStream])
|
||||
def run(self, urls: list[str]):
|
||||
"""
|
||||
Fetches content from a list of URLs and returns a list of extracted content streams.
|
||||
|
||||
@@ -211,7 +211,7 @@ class LinkContentFetcher:
|
||||
In all other scenarios, any retrieval errors are logged, and a list of successfully retrieved `ByteStream`
|
||||
objects is returned.
|
||||
"""
|
||||
streams: List[ByteStream] = []
|
||||
streams: list[ByteStream] = []
|
||||
if not urls:
|
||||
return {"streams": streams}
|
||||
|
||||
@@ -233,8 +233,8 @@ class LinkContentFetcher:
|
||||
|
||||
return {"streams": streams}
|
||||
|
||||
@component.output_types(streams=List[ByteStream])
|
||||
async def run_async(self, urls: List[str]):
|
||||
@component.output_types(streams=list[ByteStream])
|
||||
async def run_async(self, urls: list[str]):
|
||||
"""
|
||||
Asynchronously fetches content from a list of URLs and returns a list of extracted content streams.
|
||||
|
||||
@@ -243,7 +243,7 @@ class LinkContentFetcher:
|
||||
:param urls: A list of URLs to fetch content from.
|
||||
:returns: `ByteStream` objects representing the extracted content.
|
||||
"""
|
||||
streams: List[ByteStream] = []
|
||||
streams: list[ByteStream] = []
|
||||
if not urls:
|
||||
return {"streams": streams}
|
||||
|
||||
@@ -269,7 +269,7 @@ class LinkContentFetcher:
|
||||
# At this point, result is not an exception, so we need to cast it to the correct type for mypy
|
||||
if not isinstance(result, Exception): # Runtime check
|
||||
# Use cast to tell mypy that result is the tuple type returned by _fetch_async
|
||||
result_tuple = cast(Tuple[Optional[Dict[str, str]], Optional[ByteStream]], result)
|
||||
result_tuple = cast(tuple[Optional[dict[str, str]], Optional[ByteStream]], result)
|
||||
stream_metadata, stream = result_tuple
|
||||
if stream_metadata is not None and stream is not None:
|
||||
stream.meta.update(stream_metadata)
|
||||
@@ -278,7 +278,7 @@ class LinkContentFetcher:
|
||||
|
||||
return {"streams": streams}
|
||||
|
||||
def _fetch(self, url: str) -> Tuple[Dict[str, str], ByteStream]:
|
||||
def _fetch(self, url: str) -> tuple[dict[str, str], ByteStream]:
|
||||
"""
|
||||
Fetches content from a URL and returns it as a ByteStream.
|
||||
|
||||
@@ -312,7 +312,7 @@ class LinkContentFetcher:
|
||||
|
||||
async def _fetch_async(
|
||||
self, url: str, client: httpx.AsyncClient
|
||||
) -> Tuple[Optional[Dict[str, str]], Optional[ByteStream]]:
|
||||
) -> tuple[Optional[dict[str, str]], Optional[ByteStream]]:
|
||||
"""
|
||||
Asynchronously fetches content from a URL and returns it as a ByteStream.
|
||||
|
||||
@@ -322,7 +322,7 @@ class LinkContentFetcher:
|
||||
"""
|
||||
content_type: str = "text/html"
|
||||
stream: Optional[ByteStream] = None
|
||||
metadata: Optional[Dict[str, str]] = None
|
||||
metadata: Optional[dict[str, str]] = None
|
||||
|
||||
try:
|
||||
response = await self._get_response_async(url, client)
|
||||
@@ -342,7 +342,7 @@ class LinkContentFetcher:
|
||||
|
||||
return metadata, stream
|
||||
|
||||
def _fetch_with_exception_suppression(self, url: str) -> Tuple[Optional[Dict[str, str]], Optional[ByteStream]]:
|
||||
def _fetch_with_exception_suppression(self, url: str) -> tuple[Optional[dict[str, str]], Optional[ByteStream]]:
|
||||
"""
|
||||
Fetches content from a URL and returns it as a ByteStream.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from openai.lib.azure import AzureADTokenProvider, AzureOpenAI
|
||||
|
||||
@@ -67,9 +67,9 @@ class AzureOpenAIGenerator(OpenAIGenerator):
|
||||
system_prompt: Optional[str] = None,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
default_headers: Optional[Dict[str, str]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
default_headers: Optional[dict[str, str]] = None,
|
||||
*,
|
||||
azure_ad_token_provider: Optional[AzureADTokenProvider] = None,
|
||||
):
|
||||
@@ -164,7 +164,7 @@ class AzureOpenAIGenerator(OpenAIGenerator):
|
||||
default_headers=self.default_headers,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -194,7 +194,7 @@ class AzureOpenAIGenerator(OpenAIGenerator):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from openai.lib.azure import AsyncAzureADTokenProvider, AsyncAzureOpenAI, AzureADTokenProvider, AzureOpenAI
|
||||
|
||||
@@ -80,13 +80,13 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
default_headers: Optional[Dict[str, str]] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
default_headers: Optional[dict[str, str]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tools_strict: bool = False,
|
||||
*,
|
||||
azure_ad_token_provider: Optional[Union[AzureADTokenProvider, AsyncAzureADTokenProvider]] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the Azure OpenAI Chat Generator component.
|
||||
@@ -170,7 +170,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
|
||||
self.tools = tools
|
||||
self.tools_strict = tools_strict
|
||||
|
||||
client_args: Dict[str, Any] = {
|
||||
client_args: dict[str, Any] = {
|
||||
"api_version": api_version,
|
||||
"azure_endpoint": azure_endpoint,
|
||||
"azure_deployment": azure_deployment,
|
||||
@@ -190,7 +190,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_args
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -221,7 +221,7 @@ class AzureOpenAIChatGenerator(OpenAIChatGenerator):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AzureOpenAIChatGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AzureOpenAIChatGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Union
|
||||
from typing import Any, AsyncIterable, Iterable, Optional, Union
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message
|
||||
@@ -47,7 +47,7 @@ with LazyImport(message="Run 'pip install \"huggingface_hub[inference]>=0.27.0\"
|
||||
)
|
||||
|
||||
|
||||
def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[List["ChatCompletionOutputToolCall"]]) -> List[ToolCall]:
|
||||
def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[list["ChatCompletionOutputToolCall"]]) -> list[ToolCall]:
|
||||
"""
|
||||
Convert HuggingFace API tool calls to a list of Haystack ToolCall.
|
||||
|
||||
@@ -93,8 +93,8 @@ def _convert_hfapi_tool_calls(hfapi_tool_calls: Optional[List["ChatCompletionOut
|
||||
|
||||
|
||||
def _convert_tools_to_hfapi_tools(
|
||||
tools: Optional[Union[List[Tool], Toolset]],
|
||||
) -> Optional[List["ChatCompletionInputTool"]]:
|
||||
tools: Optional[Union[list[Tool], Toolset]],
|
||||
) -> Optional[list["ChatCompletionInputTool"]]:
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
@@ -140,7 +140,7 @@ def _map_hf_finish_reason_to_haystack(choice: "ChatCompletionStreamOutputChoice"
|
||||
return "tool_calls"
|
||||
|
||||
# Map HuggingFace finish reasons to Haystack standard ones
|
||||
mapping: Dict[str, FinishReason] = {
|
||||
mapping: dict[str, FinishReason] = {
|
||||
"length": "length", # Direct match
|
||||
"eos_token": "stop", # EOS token means natural stop
|
||||
"stop_sequence": "stop", # Stop sequence means natural stop
|
||||
@@ -151,7 +151,7 @@ def _map_hf_finish_reason_to_haystack(choice: "ChatCompletionStreamOutputChoice"
|
||||
|
||||
def _convert_chat_completion_stream_output_to_streaming_chunk(
|
||||
chunk: "ChatCompletionStreamOutput",
|
||||
previous_chunks: List[StreamingChunk],
|
||||
previous_chunks: list[StreamingChunk],
|
||||
component_info: Optional[ComponentInfo] = None,
|
||||
) -> StreamingChunk:
|
||||
"""
|
||||
@@ -287,12 +287,12 @@ class HuggingFaceAPIChatGenerator:
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
api_type: Union[HFGenerationAPIType, str],
|
||||
api_params: Dict[str, str],
|
||||
api_params: dict[str, str],
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
stop_words: Optional[List[str]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
stop_words: Optional[list[str]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the HuggingFaceAPIChatGenerator instance.
|
||||
@@ -372,7 +372,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
self.generation_kwargs = generation_kwargs
|
||||
self.streaming_callback = streaming_callback
|
||||
|
||||
resolved_api_params: Dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
|
||||
resolved_api_params: dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
|
||||
self._client = InferenceClient(
|
||||
model_or_url, token=token.resolve_value() if token else None, **resolved_api_params
|
||||
)
|
||||
@@ -381,7 +381,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
)
|
||||
self.tools = tools
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -400,7 +400,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIChatGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIChatGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
"""
|
||||
@@ -412,12 +412,12 @@ class HuggingFaceAPIChatGenerator:
|
||||
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
messages: list[ChatMessage],
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
):
|
||||
"""
|
||||
@@ -463,12 +463,12 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
return self._run_non_streaming(formatted_messages, generation_kwargs, hf_tools)
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
async def run_async(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
messages: list[ChatMessage],
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
):
|
||||
"""
|
||||
@@ -517,8 +517,8 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
def _run_streaming(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
generation_kwargs: Dict[str, Any],
|
||||
messages: list[dict[str, str]],
|
||||
generation_kwargs: dict[str, Any],
|
||||
streaming_callback: SyncStreamingCallbackT,
|
||||
):
|
||||
api_output: Iterable[ChatCompletionStreamOutput] = self._client.chat_completion(
|
||||
@@ -529,7 +529,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
)
|
||||
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
streaming_chunks: List[StreamingChunk] = []
|
||||
streaming_chunks: list[StreamingChunk] = []
|
||||
for chunk in api_output:
|
||||
streaming_chunk = _convert_chat_completion_stream_output_to_streaming_chunk(
|
||||
chunk=chunk, previous_chunks=streaming_chunks, component_info=component_info
|
||||
@@ -545,10 +545,10 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
def _run_non_streaming(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
generation_kwargs: Dict[str, Any],
|
||||
tools: Optional[List["ChatCompletionInputTool"]] = None,
|
||||
) -> Dict[str, List[ChatMessage]]:
|
||||
messages: list[dict[str, str]],
|
||||
generation_kwargs: dict[str, Any],
|
||||
tools: Optional[list["ChatCompletionInputTool"]] = None,
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
api_chat_output: ChatCompletionOutput = self._client.chat_completion(
|
||||
messages=messages, tools=tools, **generation_kwargs
|
||||
)
|
||||
@@ -565,7 +565,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
tool_calls = _convert_hfapi_tool_calls(choice.message.tool_calls)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
meta: dict[str, Any] = {
|
||||
"model": self._client.model,
|
||||
"finish_reason": choice.finish_reason,
|
||||
"index": choice.index,
|
||||
@@ -584,8 +584,8 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
async def _run_streaming_async(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
generation_kwargs: Dict[str, Any],
|
||||
messages: list[dict[str, str]],
|
||||
generation_kwargs: dict[str, Any],
|
||||
streaming_callback: AsyncStreamingCallbackT,
|
||||
):
|
||||
api_output: AsyncIterable[ChatCompletionStreamOutput] = await self._async_client.chat_completion(
|
||||
@@ -596,7 +596,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
)
|
||||
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
streaming_chunks: List[StreamingChunk] = []
|
||||
streaming_chunks: list[StreamingChunk] = []
|
||||
async for chunk in api_output:
|
||||
stream_chunk = _convert_chat_completion_stream_output_to_streaming_chunk(
|
||||
chunk=chunk, previous_chunks=streaming_chunks, component_info=component_info
|
||||
@@ -612,10 +612,10 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
async def _run_non_streaming_async(
|
||||
self,
|
||||
messages: List[Dict[str, str]],
|
||||
generation_kwargs: Dict[str, Any],
|
||||
tools: Optional[List["ChatCompletionInputTool"]] = None,
|
||||
) -> Dict[str, List[ChatMessage]]:
|
||||
messages: list[dict[str, str]],
|
||||
generation_kwargs: dict[str, Any],
|
||||
tools: Optional[list["ChatCompletionInputTool"]] = None,
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
api_chat_output: ChatCompletionOutput = await self._async_client.chat_completion(
|
||||
messages=messages, tools=tools, **generation_kwargs
|
||||
)
|
||||
@@ -629,7 +629,7 @@ class HuggingFaceAPIChatGenerator:
|
||||
|
||||
tool_calls = _convert_hfapi_tool_calls(choice.message.tool_calls)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
meta: dict[str, Any] = {
|
||||
"model": self._async_client.model,
|
||||
"finish_reason": choice.finish_reason,
|
||||
"index": choice.index,
|
||||
|
||||
@@ -8,7 +8,7 @@ import re
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional, Union, cast
|
||||
from typing import Any, Callable, Literal, Optional, Union, cast
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.dataclasses import AsyncStreamingCallbackT, ChatMessage, ComponentInfo, StreamingCallbackT, ToolCall
|
||||
@@ -57,7 +57,7 @@ DEFAULT_TOOL_PATTERN = (
|
||||
)
|
||||
|
||||
|
||||
def default_tool_parser(text: str) -> Optional[List[ToolCall]]:
|
||||
def default_tool_parser(text: str) -> Optional[list[ToolCall]]:
|
||||
"""
|
||||
Default implementation for parsing tool calls from model output text.
|
||||
|
||||
@@ -131,12 +131,12 @@ class HuggingFaceLocalChatGenerator:
|
||||
device: Optional[ComponentDevice] = None,
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
chat_template: Optional[str] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
|
||||
stop_words: Optional[List[str]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
|
||||
stop_words: Optional[list[str]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tool_parsing_function: Optional[Callable[[str], Optional[List[ToolCall]]]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tool_parsing_function: Optional[Callable[[str], Optional[list[ToolCall]]]] = None,
|
||||
async_executor: Optional[ThreadPoolExecutor] = None,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -263,7 +263,7 @@ class HuggingFaceLocalChatGenerator:
|
||||
if self._owns_executor:
|
||||
self.executor.shutdown(wait=True)
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
@@ -278,7 +278,7 @@ class HuggingFaceLocalChatGenerator:
|
||||
if self.pipeline is None:
|
||||
self.pipeline = pipeline(**self.huggingface_pipeline_kwargs)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -304,7 +304,7 @@ class HuggingFaceLocalChatGenerator:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceLocalChatGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceLocalChatGenerator":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -329,14 +329,14 @@ class HuggingFaceLocalChatGenerator:
|
||||
deserialize_hf_model_kwargs(huggingface_pipeline_kwargs)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
messages: list[ChatMessage],
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
) -> Dict[str, List[ChatMessage]]:
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
"""
|
||||
Invoke text generation inference based on the provided messages and generation parameters.
|
||||
|
||||
@@ -380,7 +380,7 @@ class HuggingFaceLocalChatGenerator:
|
||||
index: int,
|
||||
tokenizer: Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"],
|
||||
prompt: str,
|
||||
generation_kwargs: Dict[str, Any],
|
||||
generation_kwargs: dict[str, Any],
|
||||
parse_tool_calls: bool = False,
|
||||
) -> ChatMessage:
|
||||
"""
|
||||
@@ -424,7 +424,7 @@ class HuggingFaceLocalChatGenerator:
|
||||
return ChatMessage.from_assistant(tool_calls=tool_calls, text=None if tool_calls else text, meta=meta)
|
||||
|
||||
@staticmethod
|
||||
def _validate_stop_words(stop_words: Optional[List[str]]) -> Optional[List[str]]:
|
||||
def _validate_stop_words(stop_words: Optional[list[str]]) -> Optional[list[str]]:
|
||||
"""
|
||||
Validates the provided stop words.
|
||||
|
||||
@@ -441,14 +441,14 @@ class HuggingFaceLocalChatGenerator:
|
||||
|
||||
return list(set(stop_words or []))
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
async def run_async(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
messages: list[ChatMessage],
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
) -> Dict[str, List[ChatMessage]]:
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
"""
|
||||
Asynchronously invokes text generation inference based on the provided messages and generation parameters.
|
||||
|
||||
@@ -518,11 +518,11 @@ class HuggingFaceLocalChatGenerator:
|
||||
|
||||
def _prepare_inputs(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
messages: list[ChatMessage],
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Prepares the inputs for the Hugging Face pipeline.
|
||||
|
||||
@@ -601,13 +601,13 @@ class HuggingFaceLocalChatGenerator:
|
||||
def _convert_hf_output_to_chat_messages(
|
||||
self,
|
||||
*,
|
||||
hf_pipeline_output: List[Dict[str, Any]],
|
||||
hf_pipeline_output: list[dict[str, Any]],
|
||||
prepared_prompt: str,
|
||||
tokenizer: Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"],
|
||||
generation_kwargs: Dict[str, Any],
|
||||
stop_words: Optional[List[str]],
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
) -> List[ChatMessage]:
|
||||
generation_kwargs: dict[str, Any],
|
||||
stop_words: Optional[list[str]],
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
) -> list[ChatMessage]:
|
||||
"""
|
||||
Converts the HuggingFace pipeline output into a List of ChatMessages
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
|
||||
@@ -90,12 +90,12 @@ class OpenAIChatGenerator:
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
api_base_url: Optional[str] = None,
|
||||
organization: Optional[str] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tools_strict: bool = False,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an instance of OpenAIChatGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-4o-mini
|
||||
@@ -168,7 +168,7 @@ class OpenAIChatGenerator:
|
||||
if max_retries is None:
|
||||
max_retries = int(os.environ.get("OPENAI_MAX_RETRIES", "5"))
|
||||
|
||||
client_kwargs: Dict[str, Any] = {
|
||||
client_kwargs: dict[str, Any] = {
|
||||
"api_key": api_key.resolve_value(),
|
||||
"organization": organization,
|
||||
"base_url": api_base_url,
|
||||
@@ -181,13 +181,13 @@ class OpenAIChatGenerator:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=True), **client_kwargs
|
||||
)
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -211,7 +211,7 @@ class OpenAIChatGenerator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIChatGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAIChatGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
@@ -227,14 +227,14 @@ class OpenAIChatGenerator:
|
||||
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
messages: list[ChatMessage],
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
*,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tools_strict: Optional[bool] = None,
|
||||
):
|
||||
"""
|
||||
@@ -299,14 +299,14 @@ class OpenAIChatGenerator:
|
||||
|
||||
return {"replies": completions}
|
||||
|
||||
@component.output_types(replies=List[ChatMessage])
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
async def run_async(
|
||||
self,
|
||||
messages: List[ChatMessage],
|
||||
messages: list[ChatMessage],
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
*,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tools_strict: Optional[bool] = None,
|
||||
):
|
||||
"""
|
||||
@@ -380,12 +380,12 @@ class OpenAIChatGenerator:
|
||||
def _prepare_api_call( # noqa: PLR0913
|
||||
self,
|
||||
*,
|
||||
messages: List[ChatMessage],
|
||||
messages: list[ChatMessage],
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tools: Optional[Union[List[Tool], Toolset]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: Optional[Union[list[Tool], Toolset]] = None,
|
||||
tools_strict: Optional[bool] = None,
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
# update generation kwargs by merging with the generation kwargs passed to the run method
|
||||
generation_kwargs = {**self.generation_kwargs, **(generation_kwargs or {})}
|
||||
|
||||
@@ -423,9 +423,9 @@ class OpenAIChatGenerator:
|
||||
**generation_kwargs,
|
||||
}
|
||||
|
||||
def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> List[ChatMessage]:
|
||||
def _handle_stream_response(self, chat_completion: Stream, callback: SyncStreamingCallbackT) -> list[ChatMessage]:
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
chunks: List[StreamingChunk] = []
|
||||
chunks: list[StreamingChunk] = []
|
||||
for chunk in chat_completion: # pylint: disable=not-an-iterable
|
||||
assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
|
||||
chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
|
||||
@@ -437,9 +437,9 @@ class OpenAIChatGenerator:
|
||||
|
||||
async def _handle_async_stream_response(
|
||||
self, chat_completion: AsyncStream, callback: AsyncStreamingCallbackT
|
||||
) -> List[ChatMessage]:
|
||||
) -> list[ChatMessage]:
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
chunks: List[StreamingChunk] = []
|
||||
chunks: list[StreamingChunk] = []
|
||||
async for chunk in chat_completion: # pylint: disable=not-an-iterable
|
||||
assert len(chunk.choices) <= 1, "Streaming responses should have at most one choice."
|
||||
chunk_delta = _convert_chat_completion_chunk_to_streaming_chunk(
|
||||
@@ -450,7 +450,7 @@ class OpenAIChatGenerator:
|
||||
return [_convert_streaming_chunks_to_chat_message(chunks=chunks)]
|
||||
|
||||
|
||||
def _check_finish_reason(meta: Dict[str, Any]) -> None:
|
||||
def _check_finish_reason(meta: dict[str, Any]) -> None:
|
||||
if meta["finish_reason"] == "length":
|
||||
logger.warning(
|
||||
"The completion for index {index} has been truncated before reaching a natural stopping point. "
|
||||
@@ -507,7 +507,7 @@ def _convert_chat_completion_to_chat_message(completion: ChatCompletion, choice:
|
||||
|
||||
|
||||
def _convert_chat_completion_chunk_to_streaming_chunk(
|
||||
chunk: ChatCompletionChunk, previous_chunks: List[StreamingChunk], component_info: Optional[ComponentInfo] = None
|
||||
chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: Optional[ComponentInfo] = None
|
||||
) -> StreamingChunk:
|
||||
"""
|
||||
Converts the streaming response chunk from the OpenAI API to a StreamingChunk.
|
||||
@@ -520,7 +520,7 @@ def _convert_chat_completion_chunk_to_streaming_chunk(
|
||||
:returns:
|
||||
A StreamingChunk object representing the content of the chunk from the OpenAI API.
|
||||
"""
|
||||
finish_reason_mapping: Dict[str, FinishReason] = {
|
||||
finish_reason_mapping: dict[str, FinishReason] = {
|
||||
"stop": "stop",
|
||||
"length": "length",
|
||||
"content_filter": "content_filter",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Protocol, TypeVar
|
||||
from typing import Any, Protocol, TypeVar
|
||||
|
||||
from haystack.dataclasses import ChatMessage
|
||||
|
||||
@@ -21,12 +21,12 @@ class ChatGenerator(Protocol):
|
||||
responses using a Language Model. They return a dictionary.
|
||||
"""
|
||||
|
||||
def run(self, messages: List[ChatMessage]) -> Dict[str, Any]:
|
||||
def run(self, messages: list[ChatMessage]) -> dict[str, Any]:
|
||||
"""
|
||||
Generate messages using the underlying Language Model.
|
||||
|
||||
Implementing classes may accept additional optional parameters in their run method.
|
||||
For example: `def run (self, messages: List[ChatMessage], param_a="default", param_b="another_default")`.
|
||||
For example: `def run (self, messages: list[ChatMessage], param_a="default", param_b="another_default")`.
|
||||
|
||||
:param messages:
|
||||
A list of ChatMessage instances representing the input messages.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union, cast
|
||||
from typing import Any, Iterable, Optional, Union, cast
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.dataclasses import (
|
||||
@@ -95,10 +95,10 @@ class HuggingFaceAPIGenerator:
|
||||
def __init__( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
api_type: Union[HFGenerationAPIType, str],
|
||||
api_params: Dict[str, str],
|
||||
api_params: dict[str, str],
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
stop_words: Optional[List[str]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
stop_words: Optional[list[str]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
):
|
||||
"""
|
||||
@@ -173,12 +173,12 @@ class HuggingFaceAPIGenerator:
|
||||
self.generation_kwargs = generation_kwargs
|
||||
self.streaming_callback = streaming_callback
|
||||
|
||||
resolved_api_params: Dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
|
||||
resolved_api_params: dict[str, Any] = {k: v for k, v in api_params.items() if k != "model" and k != "url"}
|
||||
self._client = InferenceClient(
|
||||
model_or_url, token=token.resolve_value() if token else None, **resolved_api_params
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -196,7 +196,7 @@ class HuggingFaceAPIGenerator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceAPIGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceAPIGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
"""
|
||||
@@ -207,12 +207,12 @@ class HuggingFaceAPIGenerator:
|
||||
init_params["streaming_callback"] = deserialize_callable(serialized_callback_handler)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[str], meta=List[Dict[str, Any]])
|
||||
@component.output_types(replies=list[str], meta=list[dict[str, Any]])
|
||||
def run(
|
||||
self,
|
||||
prompt: str,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Invoke the text generation inference for the given prompt and generation parameters.
|
||||
@@ -251,7 +251,7 @@ class HuggingFaceAPIGenerator:
|
||||
def _stream_and_build_response(
|
||||
self, hf_output: Iterable["TextGenerationStreamOutput"], streaming_callback: SyncStreamingCallbackT
|
||||
):
|
||||
chunks: List[StreamingChunk] = []
|
||||
chunks: list[StreamingChunk] = []
|
||||
first_chunk_time = None
|
||||
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
@@ -264,7 +264,7 @@ class HuggingFaceAPIGenerator:
|
||||
if first_chunk_time is None:
|
||||
first_chunk_time = datetime.now().isoformat()
|
||||
|
||||
mapping: Dict[str, FinishReason] = {
|
||||
mapping: dict[str, FinishReason] = {
|
||||
"length": "length", # Direct match
|
||||
"eos_token": "stop", # EOS token means natural stop
|
||||
"stop_sequence": "stop", # Stop sequence means natural stop
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional, cast
|
||||
from typing import Any, Literal, Optional, cast
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict, logging
|
||||
from haystack.dataclasses import ComponentInfo, StreamingCallbackT, select_streaming_callback
|
||||
@@ -61,9 +61,9 @@ class HuggingFaceLocalGenerator:
|
||||
task: Optional[Literal["text-generation", "text2text-generation"]] = None,
|
||||
device: Optional[ComponentDevice] = None,
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
huggingface_pipeline_kwargs: Optional[Dict[str, Any]] = None,
|
||||
stop_words: Optional[List[str]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
huggingface_pipeline_kwargs: Optional[dict[str, Any]] = None,
|
||||
stop_words: Optional[list[str]] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
):
|
||||
"""
|
||||
@@ -131,7 +131,7 @@ class HuggingFaceLocalGenerator:
|
||||
self.stopping_criteria_list: Optional[StoppingCriteriaList] = None
|
||||
self.streaming_callback = streaming_callback
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
@@ -164,7 +164,7 @@ class HuggingFaceLocalGenerator:
|
||||
)
|
||||
self.stopping_criteria_list = StoppingCriteriaList([stop_words_criteria])
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -188,7 +188,7 @@ class HuggingFaceLocalGenerator:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceLocalGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceLocalGenerator":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -207,12 +207,12 @@ class HuggingFaceLocalGenerator:
|
||||
deserialize_hf_model_kwargs(huggingface_pipeline_kwargs)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[str])
|
||||
@component.output_types(replies=list[str])
|
||||
def run(
|
||||
self,
|
||||
prompt: str,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Run the text generation model on the given prompt.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from openai import OpenAI, Stream
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
@@ -69,10 +69,10 @@ class OpenAIGenerator:
|
||||
api_base_url: Optional[str] = None,
|
||||
organization: Optional[str] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an instance of OpenAIGenerator. Unless specified otherwise in `model`, uses OpenAI's gpt-4o-mini
|
||||
@@ -141,13 +141,13 @@ class OpenAIGenerator:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=False),
|
||||
)
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -168,7 +168,7 @@ class OpenAIGenerator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "OpenAIGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "OpenAIGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
@@ -184,13 +184,13 @@ class OpenAIGenerator:
|
||||
data["init_parameters"]["streaming_callback"] = deserialize_callable(serialized_callback_handler)
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(replies=List[str], meta=List[Dict[str, Any]])
|
||||
@component.output_types(replies=list[str], meta=list[dict[str, Any]])
|
||||
def run(
|
||||
self,
|
||||
prompt: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
streaming_callback: Optional[StreamingCallbackT] = None,
|
||||
generation_kwargs: Optional[Dict[str, Any]] = None,
|
||||
generation_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Invoke the text generation inference based on the provided messages and generation parameters.
|
||||
@@ -236,14 +236,14 @@ class OpenAIGenerator:
|
||||
**generation_kwargs,
|
||||
)
|
||||
|
||||
completions: List[ChatMessage] = []
|
||||
completions: list[ChatMessage] = []
|
||||
if streaming_callback is not None:
|
||||
num_responses = generation_kwargs.pop("n", 1)
|
||||
if num_responses > 1:
|
||||
raise ValueError("Cannot stream multiple responses, please set n=1.")
|
||||
|
||||
component_info = ComponentInfo.from_component(self)
|
||||
chunks: List[StreamingChunk] = []
|
||||
chunks: list[StreamingChunk] = []
|
||||
for chunk in completion:
|
||||
chunk_delta: StreamingChunk = _convert_chat_completion_chunk_to_streaming_chunk(
|
||||
chunk=chunk, # type: ignore
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from openai import OpenAI
|
||||
from openai.types.image import Image
|
||||
@@ -42,7 +42,7 @@ class DALLEImageGenerator:
|
||||
organization: Optional[str] = None,
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
http_client_kwargs: Optional[Dict[str, Any]] = None,
|
||||
http_client_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Creates an instance of DALLEImageGenerator. Unless specified otherwise in `model`, uses OpenAI's dall-e-3.
|
||||
@@ -94,7 +94,7 @@ class DALLEImageGenerator:
|
||||
http_client=init_http_client(self.http_client_kwargs, async_client=False),
|
||||
)
|
||||
|
||||
@component.output_types(images=List[str], revised_prompt=str)
|
||||
@component.output_types(images=list[str], revised_prompt=str)
|
||||
def run(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -136,7 +136,7 @@ class DALLEImageGenerator:
|
||||
revised_prompt = ""
|
||||
return {"images": [image_str], "revised_prompt": revised_prompt}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize this component to a dictionary.
|
||||
|
||||
@@ -156,7 +156,7 @@ class DALLEImageGenerator:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DALLEImageGenerator":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DALLEImageGenerator":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
from typing import Dict, List
|
||||
|
||||
from haystack import logging
|
||||
from haystack.dataclasses import ChatMessage, StreamingChunk, ToolCall
|
||||
@@ -69,7 +68,7 @@ def print_streaming_chunk(chunk: StreamingChunk) -> None:
|
||||
print("\n\n", flush=True, end="")
|
||||
|
||||
|
||||
def _convert_streaming_chunks_to_chat_message(chunks: List[StreamingChunk]) -> ChatMessage:
|
||||
def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> ChatMessage:
|
||||
"""
|
||||
Connects the streaming chunks into a single ChatMessage.
|
||||
|
||||
@@ -81,7 +80,7 @@ def _convert_streaming_chunks_to_chat_message(chunks: List[StreamingChunk]) -> C
|
||||
tool_calls = []
|
||||
|
||||
# Process tool calls if present in any chunk
|
||||
tool_call_data: Dict[int, Dict[str, str]] = {} # Track tool calls by index
|
||||
tool_call_data: dict[int, dict[str, str]] = {} # Track tool calls by index
|
||||
for chunk in chunks:
|
||||
if chunk.tool_calls:
|
||||
for tool_call in chunk.tool_calls:
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import itertools
|
||||
from enum import Enum
|
||||
from math import inf
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
from typing import Any, Callable, Optional, Union
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.core.component.types import Variadic
|
||||
@@ -103,16 +103,16 @@ class AnswerJoiner:
|
||||
"""
|
||||
if isinstance(join_mode, str):
|
||||
join_mode = JoinMode.from_str(join_mode)
|
||||
join_mode_functions: Dict[JoinMode, Callable[[List[List[AnswerType]]], List[AnswerType]]] = {
|
||||
join_mode_functions: dict[JoinMode, Callable[[list[list[AnswerType]]], list[AnswerType]]] = {
|
||||
JoinMode.CONCATENATE: self._concatenate
|
||||
}
|
||||
self.join_mode_function: Callable[[List[List[AnswerType]]], List[AnswerType]] = join_mode_functions[join_mode]
|
||||
self.join_mode_function: Callable[[list[list[AnswerType]]], list[AnswerType]] = join_mode_functions[join_mode]
|
||||
self.join_mode = join_mode
|
||||
self.top_k = top_k
|
||||
self.sort_by_score = sort_by_score
|
||||
|
||||
@component.output_types(answers=List[AnswerType])
|
||||
def run(self, answers: Variadic[List[AnswerType]], top_k: Optional[int] = None):
|
||||
@component.output_types(answers=list[AnswerType])
|
||||
def run(self, answers: Variadic[list[AnswerType]], top_k: Optional[int] = None):
|
||||
"""
|
||||
Joins multiple lists of Answers into a single list depending on the `join_mode` parameter.
|
||||
|
||||
@@ -128,7 +128,7 @@ class AnswerJoiner:
|
||||
"""
|
||||
answers_list = list(answers)
|
||||
join_function = self.join_mode_function
|
||||
output_answers: List[AnswerType] = join_function(answers_list)
|
||||
output_answers: list[AnswerType] = join_function(answers_list)
|
||||
|
||||
if self.sort_by_score:
|
||||
output_answers = sorted(
|
||||
@@ -140,7 +140,7 @@ class AnswerJoiner:
|
||||
output_answers = output_answers[:top_k]
|
||||
return {"answers": output_answers}
|
||||
|
||||
def _concatenate(self, answer_lists: List[List[AnswerType]]) -> List[AnswerType]:
|
||||
def _concatenate(self, answer_lists: list[list[AnswerType]]) -> list[AnswerType]:
|
||||
"""
|
||||
Concatenate multiple lists of Answers, flattening them into a single list and sorting by score.
|
||||
|
||||
@@ -148,7 +148,7 @@ class AnswerJoiner:
|
||||
"""
|
||||
return list(itertools.chain.from_iterable(answer_lists))
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -158,7 +158,7 @@ class AnswerJoiner:
|
||||
return default_to_dict(self, join_mode=str(self.join_mode), top_k=self.top_k, sort_by_score=self.sort_by_score)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AnswerJoiner":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AnswerJoiner":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, Type
|
||||
from typing import Any
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.core.component.types import GreedyVariadic
|
||||
@@ -31,7 +31,6 @@ class BranchJoiner:
|
||||
### Example Usage:
|
||||
```python
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.converters import OutputAdapter
|
||||
@@ -55,10 +54,10 @@ class BranchJoiner:
|
||||
pipe = Pipeline()
|
||||
|
||||
# Add components to the pipeline
|
||||
pipe.add_component('joiner', BranchJoiner(List[ChatMessage]))
|
||||
pipe.add_component('joiner', BranchJoiner(list[ChatMessage]))
|
||||
pipe.add_component('generator', OpenAIChatGenerator(model="gpt-4o-mini"))
|
||||
pipe.add_component('validator', JsonSchemaValidator(json_schema=person_schema))
|
||||
pipe.add_component('adapter', OutputAdapter("{{chat_message}}", List[ChatMessage], unsafe=True))
|
||||
pipe.add_component('adapter', OutputAdapter("{{chat_message}}", list[ChatMessage], unsafe=True))
|
||||
|
||||
# And connect them
|
||||
pipe.connect("adapter", "joiner")
|
||||
@@ -80,16 +79,16 @@ class BranchJoiner:
|
||||
```
|
||||
|
||||
Note that `BranchJoiner` can manage only one data type at a time. In this case, `BranchJoiner` is created for
|
||||
passing `List[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
|
||||
passing `list[ChatMessage]`. This determines the type of data that `BranchJoiner` will receive from the upstream
|
||||
connected components and also the type of data that `BranchJoiner` will send through its output.
|
||||
|
||||
In the code example, `BranchJoiner` receives a looped back `List[ChatMessage]` from the `JsonSchemaValidator` and
|
||||
In the code example, `BranchJoiner` receives a looped back `list[ChatMessage]` from the `JsonSchemaValidator` and
|
||||
sends it down to the `OpenAIChatGenerator` for re-generation. We can have multiple loopback connections in the
|
||||
pipeline. In this instance, the downstream component is only one (the `OpenAIChatGenerator`), but the pipeline could
|
||||
have more than one downstream component.
|
||||
"""
|
||||
|
||||
def __init__(self, type_: Type):
|
||||
def __init__(self, type_: type):
|
||||
"""
|
||||
Creates a `BranchJoiner` component.
|
||||
|
||||
@@ -99,7 +98,7 @@ class BranchJoiner:
|
||||
component.set_input_types(self, value=GreedyVariadic[type_]) # type: ignore
|
||||
component.set_output_types(self, value=type_)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component into a dictionary.
|
||||
|
||||
@@ -109,7 +108,7 @@ class BranchJoiner:
|
||||
return default_to_dict(self, type_=serialize_type(self.type_))
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "BranchJoiner":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "BranchJoiner":
|
||||
"""
|
||||
Deserializes a `BranchJoiner` instance from a dictionary.
|
||||
|
||||
@@ -120,7 +119,7 @@ class BranchJoiner:
|
||||
data["init_parameters"]["type_"] = deserialize_type(data["init_parameters"]["type_"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def run(self, **kwargs) -> Dict[str, Any]:
|
||||
def run(self, **kwargs) -> dict[str, Any]:
|
||||
"""
|
||||
Executes the `BranchJoiner`, selecting the first available input value and passing it downstream.
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import itertools
|
||||
from collections import defaultdict
|
||||
from enum import Enum
|
||||
from math import inf
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.core.component.types import Variadic
|
||||
@@ -87,7 +87,7 @@ class DocumentJoiner:
|
||||
def __init__(
|
||||
self,
|
||||
join_mode: Union[str, JoinMode] = JoinMode.CONCATENATE,
|
||||
weights: Optional[List[float]] = None,
|
||||
weights: Optional[list[float]] = None,
|
||||
top_k: Optional[int] = None,
|
||||
sort_by_score: bool = True,
|
||||
):
|
||||
@@ -126,8 +126,8 @@ class DocumentJoiner:
|
||||
self.top_k = top_k
|
||||
self.sort_by_score = sort_by_score
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: Variadic[List[Document]], top_k: Optional[int] = None):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: Variadic[list[Document]], top_k: Optional[int] = None):
|
||||
"""
|
||||
Joins multiple lists of Documents into a single list depending on the `join_mode` parameter.
|
||||
|
||||
@@ -161,7 +161,7 @@ class DocumentJoiner:
|
||||
return {"documents": output_documents}
|
||||
|
||||
@staticmethod
|
||||
def _concatenate(document_lists: List[List[Document]]) -> List[Document]:
|
||||
def _concatenate(document_lists: list[list[Document]]) -> list[Document]:
|
||||
"""
|
||||
Concatenate multiple lists of Documents and return only the Document with the highest score for duplicates.
|
||||
"""
|
||||
@@ -174,7 +174,7 @@ class DocumentJoiner:
|
||||
output.append(doc_with_best_score)
|
||||
return output
|
||||
|
||||
def _merge(self, document_lists: List[List[Document]]) -> List[Document]:
|
||||
def _merge(self, document_lists: list[list[Document]]) -> list[Document]:
|
||||
"""
|
||||
Merge multiple lists of Documents and calculate a weighted sum of the scores of duplicate Documents.
|
||||
"""
|
||||
@@ -196,7 +196,7 @@ class DocumentJoiner:
|
||||
|
||||
return list(documents_map.values())
|
||||
|
||||
def _reciprocal_rank_fusion(self, document_lists: List[List[Document]]) -> List[Document]:
|
||||
def _reciprocal_rank_fusion(self, document_lists: list[list[Document]]) -> list[Document]:
|
||||
"""
|
||||
Merge multiple lists of Documents and assign scores based on reciprocal rank fusion.
|
||||
|
||||
@@ -230,7 +230,7 @@ class DocumentJoiner:
|
||||
return list(documents_map.values())
|
||||
|
||||
@staticmethod
|
||||
def _distribution_based_rank_fusion(document_lists: List[List[Document]]) -> List[Document]:
|
||||
def _distribution_based_rank_fusion(document_lists: list[list[Document]]) -> list[Document]:
|
||||
"""
|
||||
Merge multiple lists of Documents and assign scores based on Distribution-Based Score Fusion.
|
||||
|
||||
@@ -260,7 +260,7 @@ class DocumentJoiner:
|
||||
|
||||
return output
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -276,7 +276,7 @@ class DocumentJoiner:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DocumentJoiner":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DocumentJoiner":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from itertools import chain
|
||||
from typing import Any, Dict, List, Optional, Type
|
||||
from typing import Any, Optional
|
||||
|
||||
from haystack import component, default_from_dict, default_to_dict
|
||||
from haystack.core.component.types import Variadic
|
||||
@@ -25,7 +25,6 @@ class ListJoiner:
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack import Pipeline
|
||||
from haystack.components.joiners import ListJoiner
|
||||
from typing import List
|
||||
|
||||
|
||||
user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")]
|
||||
@@ -48,7 +47,7 @@ class ListJoiner:
|
||||
pipe.add_component("llm", llm)
|
||||
pipe.add_component("feedback_prompt_builder", feedback_prompt_builder)
|
||||
pipe.add_component("feedback_llm", feedback_llm)
|
||||
pipe.add_component("list_joiner", ListJoiner(List[ChatMessage]))
|
||||
pipe.add_component("list_joiner", ListJoiner(list[ChatMessage]))
|
||||
|
||||
pipe.connect("prompt_builder.prompt", "llm.messages")
|
||||
pipe.connect("prompt_builder.prompt", "list_joiner")
|
||||
@@ -65,11 +64,11 @@ class ListJoiner:
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self, list_type_: Optional[Type] = None):
|
||||
def __init__(self, list_type_: Optional[type] = None):
|
||||
"""
|
||||
Creates a ListJoiner component.
|
||||
|
||||
:param list_type_: The expected type of the lists this component will join (e.g., List[ChatMessage]).
|
||||
:param list_type_: The expected type of the lists this component will join (e.g., list[ChatMessage]).
|
||||
If specified, all input lists must conform to this type. If None, the component defaults to handling
|
||||
lists of any type including mixed types.
|
||||
"""
|
||||
@@ -77,9 +76,9 @@ class ListJoiner:
|
||||
if list_type_ is not None:
|
||||
component.set_output_types(self, values=list_type_)
|
||||
else:
|
||||
component.set_output_types(self, values=List[Any])
|
||||
component.set_output_types(self, values=list[Any])
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -90,7 +89,7 @@ class ListJoiner:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ListJoiner":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "ListJoiner":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -102,7 +101,7 @@ class ListJoiner:
|
||||
data["init_parameters"]["list_type_"] = deserialize_type(data["init_parameters"]["list_type_"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def run(self, values: Variadic[List[Any]]) -> Dict[str, List[Any]]:
|
||||
def run(self, values: Variadic[list[Any]]) -> dict[str, list[Any]]:
|
||||
"""
|
||||
Joins multiple lists into a single flat list.
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import List
|
||||
|
||||
from haystack import component
|
||||
from haystack.core.component.types import Variadic
|
||||
@@ -40,7 +39,7 @@ class StringJoiner:
|
||||
```
|
||||
"""
|
||||
|
||||
@component.output_types(strings=List[str])
|
||||
@component.output_types(strings=list[str])
|
||||
def run(self, strings: Variadic[str]):
|
||||
"""
|
||||
Joins strings into a list of strings
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from copy import deepcopy
|
||||
from io import StringIO
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -54,8 +54,8 @@ class CSVDocumentCleaner:
|
||||
self.keep_id = keep_id
|
||||
pandas_import.check()
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Cleans CSV documents by removing empty rows and columns while preserving specified ignored rows and columns.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from io import StringIO
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, get_args
|
||||
from typing import Any, Literal, Optional, get_args
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -32,7 +32,7 @@ class CSVDocumentSplitter:
|
||||
self,
|
||||
row_split_threshold: Optional[int] = 2,
|
||||
column_split_threshold: Optional[int] = 2,
|
||||
read_csv_kwargs: Optional[Dict[str, Any]] = None,
|
||||
read_csv_kwargs: Optional[dict[str, Any]] = None,
|
||||
split_mode: SplitMode = "threshold",
|
||||
) -> None:
|
||||
"""
|
||||
@@ -70,8 +70,8 @@ class CSVDocumentSplitter:
|
||||
self.read_csv_kwargs = read_csv_kwargs or {}
|
||||
self.split_mode = split_mode
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Processes and splits a list of CSV documents into multiple sub-tables.
|
||||
|
||||
@@ -162,7 +162,7 @@ class CSVDocumentSplitter:
|
||||
@staticmethod
|
||||
def _find_split_indices(
|
||||
df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
|
||||
) -> List[Tuple[int, int]]:
|
||||
) -> list[tuple[int, int]]:
|
||||
"""
|
||||
Finds the indices of consecutive empty rows or columns in a DataFrame.
|
||||
|
||||
@@ -202,7 +202,7 @@ class CSVDocumentSplitter:
|
||||
|
||||
def _split_dataframe(
|
||||
self, df: "pd.DataFrame", split_threshold: int, axis: Literal["row", "column"]
|
||||
) -> List["pd.DataFrame"]:
|
||||
) -> list["pd.DataFrame"]:
|
||||
"""
|
||||
Splits a DataFrame into sub-tables based on consecutive empty rows or columns exceeding `split_threshold`.
|
||||
|
||||
@@ -237,7 +237,7 @@ class CSVDocumentSplitter:
|
||||
|
||||
def _recursive_split(
|
||||
self, df: "pd.DataFrame", row_split_threshold: int, column_split_threshold: int
|
||||
) -> List["pd.DataFrame"]:
|
||||
) -> list["pd.DataFrame"]:
|
||||
"""
|
||||
Recursively splits a DataFrame.
|
||||
|
||||
@@ -272,7 +272,7 @@ class CSVDocumentSplitter:
|
||||
|
||||
return result
|
||||
|
||||
def _split_by_row(self, df: "pd.DataFrame") -> List["pd.DataFrame"]:
|
||||
def _split_by_row(self, df: "pd.DataFrame") -> list["pd.DataFrame"]:
|
||||
"""Split each CSV row into a separate subtable"""
|
||||
split_dfs = []
|
||||
for idx, row in enumerate(df.itertuples(index=False)):
|
||||
|
||||
@@ -6,7 +6,7 @@ import re
|
||||
from copy import deepcopy
|
||||
from functools import partial, reduce
|
||||
from itertools import chain
|
||||
from typing import Generator, List, Literal, Optional, Set
|
||||
from typing import Generator, Literal, Optional
|
||||
from unicodedata import normalize
|
||||
|
||||
from haystack import Document, component, logging
|
||||
@@ -44,7 +44,7 @@ class DocumentCleaner:
|
||||
remove_extra_whitespaces: bool = True,
|
||||
remove_repeated_substrings: bool = False,
|
||||
keep_id: bool = False,
|
||||
remove_substrings: Optional[List[str]] = None,
|
||||
remove_substrings: Optional[list[str]] = None,
|
||||
remove_regex: Optional[str] = None,
|
||||
unicode_normalization: Optional[Literal["NFC", "NFKC", "NFD", "NFKD"]] = None,
|
||||
ascii_only: bool = False,
|
||||
@@ -89,8 +89,8 @@ class DocumentCleaner:
|
||||
if unicode_normalization and unicode_normalization not in ["NFC", "NFKC", "NFD", "NFKD"]:
|
||||
raise ValueError("unicode_normalization must be one of 'NFC', 'NFKC', 'NFD', 'NFKD'.")
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Cleans up the documents.
|
||||
|
||||
@@ -204,7 +204,7 @@ class DocumentCleaner:
|
||||
cleaned_text = [re.sub(regex, "", text).strip() for text in texts]
|
||||
return "\f".join(cleaned_text)
|
||||
|
||||
def _remove_substrings(self, text: str, substrings: List[str]) -> str:
|
||||
def _remove_substrings(self, text: str, substrings: list[str]) -> str:
|
||||
"""
|
||||
Remove all specified substrings from the text.
|
||||
|
||||
@@ -287,7 +287,7 @@ class DocumentCleaner:
|
||||
|
||||
return ngrams
|
||||
|
||||
def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> Set[str]:
|
||||
def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> set[str]:
|
||||
"""
|
||||
Generates all possible ngrams from a given sequence of text.
|
||||
|
||||
@@ -303,7 +303,7 @@ class DocumentCleaner:
|
||||
res = set(chain.from_iterable(ngrams))
|
||||
return res
|
||||
|
||||
def _find_longest_common_ngram(self, sequences: List[str], min_ngram: int = 3, max_ngram: int = 30) -> str:
|
||||
def _find_longest_common_ngram(self, sequences: list[str], min_ngram: int = 3, max_ngram: int = 30) -> str:
|
||||
"""
|
||||
Find the longest common ngram across a list of text sequences (e.g. start of pages).
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional
|
||||
|
||||
from haystack import Document, Pipeline, default_from_dict, default_to_dict, super_component
|
||||
from haystack.components.preprocessors.document_cleaner import DocumentCleaner
|
||||
@@ -38,7 +38,7 @@ class DocumentPreprocessor:
|
||||
split_length: int = 250,
|
||||
split_overlap: int = 0,
|
||||
split_threshold: int = 0,
|
||||
splitting_function: Optional[Callable[[str], List[str]]] = None,
|
||||
splitting_function: Optional[Callable[[str], list[str]]] = None,
|
||||
respect_sentence_boundary: bool = False,
|
||||
language: Language = "en",
|
||||
use_split_rules: bool = True,
|
||||
@@ -48,7 +48,7 @@ class DocumentPreprocessor:
|
||||
remove_extra_whitespaces: bool = True,
|
||||
remove_repeated_substrings: bool = False,
|
||||
keep_id: bool = False,
|
||||
remove_substrings: Optional[List[str]] = None,
|
||||
remove_substrings: Optional[list[str]] = None,
|
||||
remove_regex: Optional[str] = None,
|
||||
unicode_normalization: Optional[Literal["NFC", "NFKC", "NFD", "NFKD"]] = None,
|
||||
ascii_only: bool = False,
|
||||
@@ -144,12 +144,12 @@ class DocumentPreprocessor:
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# fake method, never executed, but static analyzers will not complain about missing method
|
||||
def run(self, *, documents: List[Document]) -> dict[str, list[Document]]: # noqa: D102
|
||||
def run(self, *, documents: list[Document]) -> dict[str, list[Document]]: # noqa: D102
|
||||
...
|
||||
def warm_up(self) -> None: # noqa: D102
|
||||
...
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serialize SuperComponent to a dictionary.
|
||||
|
||||
@@ -182,7 +182,7 @@ class DocumentPreprocessor:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DocumentPreprocessor":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DocumentPreprocessor":
|
||||
"""
|
||||
Deserializes the SuperComponent from a dictionary.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
|
||||
from typing import Any, Callable, Literal, Optional
|
||||
|
||||
from more_itertools import windowed
|
||||
|
||||
@@ -57,7 +57,7 @@ class DocumentSplitter:
|
||||
split_length: int = 200,
|
||||
split_overlap: int = 0,
|
||||
split_threshold: int = 0,
|
||||
splitting_function: Optional[Callable[[str], List[str]]] = None,
|
||||
splitting_function: Optional[Callable[[str], list[str]]] = None,
|
||||
respect_sentence_boundary: bool = False,
|
||||
language: Language = "en",
|
||||
use_split_rules: bool = True,
|
||||
@@ -168,8 +168,8 @@ class DocumentSplitter:
|
||||
keep_white_spaces=True,
|
||||
)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Split documents into smaller parts.
|
||||
|
||||
@@ -194,7 +194,7 @@ class DocumentSplitter:
|
||||
if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
|
||||
raise TypeError("DocumentSplitter expects a List of Documents as input.")
|
||||
|
||||
split_docs: List[Document] = []
|
||||
split_docs: list[Document] = []
|
||||
for doc in documents:
|
||||
if doc.content is None:
|
||||
raise ValueError(
|
||||
@@ -207,7 +207,7 @@ class DocumentSplitter:
|
||||
split_docs += self._split_document(doc)
|
||||
return {"documents": split_docs}
|
||||
|
||||
def _split_document(self, doc: Document) -> List[Document]:
|
||||
def _split_document(self, doc: Document) -> list[Document]:
|
||||
if self.split_by == "sentence" or self.respect_sentence_boundary:
|
||||
return self._split_by_nltk_sentence(doc)
|
||||
|
||||
@@ -216,7 +216,7 @@ class DocumentSplitter:
|
||||
|
||||
return self._split_by_character(doc)
|
||||
|
||||
def _split_by_nltk_sentence(self, doc: Document) -> List[Document]:
|
||||
def _split_by_nltk_sentence(self, doc: Document) -> list[Document]:
|
||||
split_docs = []
|
||||
|
||||
result = self.sentence_splitter.split_sentences(doc.content) # type: ignore # None check is done in run()
|
||||
@@ -241,7 +241,7 @@ class DocumentSplitter:
|
||||
|
||||
return split_docs
|
||||
|
||||
def _split_by_character(self, doc) -> List[Document]:
|
||||
def _split_by_character(self, doc) -> list[Document]:
|
||||
split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
|
||||
units = doc.content.split(split_at)
|
||||
# Add the delimiter back to all units except the last one
|
||||
@@ -256,10 +256,10 @@ class DocumentSplitter:
|
||||
text_splits=text_splits, splits_pages=splits_pages, splits_start_idxs=splits_start_idxs, meta=metadata
|
||||
)
|
||||
|
||||
def _split_by_function(self, doc) -> List[Document]:
|
||||
def _split_by_function(self, doc) -> list[Document]:
|
||||
# the check for None is done already in the run method
|
||||
splits = self.splitting_function(doc.content) # type: ignore
|
||||
docs: List[Document] = []
|
||||
docs: list[Document] = []
|
||||
for s in splits:
|
||||
meta = deepcopy(doc.meta)
|
||||
meta["source_id"] = doc.id
|
||||
@@ -267,8 +267,8 @@ class DocumentSplitter:
|
||||
return docs
|
||||
|
||||
def _concatenate_units(
|
||||
self, elements: List[str], split_length: int, split_overlap: int, split_threshold: int
|
||||
) -> Tuple[List[str], List[int], List[int]]:
|
||||
self, elements: list[str], split_length: int, split_overlap: int, split_threshold: int
|
||||
) -> tuple[list[str], list[int], list[int]]:
|
||||
"""
|
||||
Concatenates the elements into parts of split_length units.
|
||||
|
||||
@@ -277,9 +277,9 @@ class DocumentSplitter:
|
||||
units with the last split, preventing the creation of excessively small splits.
|
||||
"""
|
||||
|
||||
text_splits: List[str] = []
|
||||
splits_pages: List[int] = []
|
||||
splits_start_idxs: List[int] = []
|
||||
text_splits: list[str] = []
|
||||
splits_pages: list[int] = []
|
||||
splits_start_idxs: list[int] = []
|
||||
cur_start_idx = 0
|
||||
cur_page = 1
|
||||
segments = windowed(elements, n=split_length, step=split_length - split_overlap)
|
||||
@@ -312,12 +312,12 @@ class DocumentSplitter:
|
||||
return text_splits, splits_pages, splits_start_idxs
|
||||
|
||||
def _create_docs_from_splits(
|
||||
self, text_splits: List[str], splits_pages: List[int], splits_start_idxs: List[int], meta: Dict[str, Any]
|
||||
) -> List[Document]:
|
||||
self, text_splits: list[str], splits_pages: list[int], splits_start_idxs: list[int], meta: dict[str, Any]
|
||||
) -> list[Document]:
|
||||
"""
|
||||
Creates Document objects from splits enriching them with page number and the metadata of the original document.
|
||||
"""
|
||||
documents: List[Document] = []
|
||||
documents: list[Document] = []
|
||||
|
||||
for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs)):
|
||||
copied_meta = deepcopy(meta)
|
||||
@@ -367,7 +367,7 @@ class DocumentSplitter:
|
||||
overlapping_range = (0, overlapping_range[1] - overlapping_range[0])
|
||||
previous_doc.meta["_split_overlap"].append({"doc_id": current_doc.id, "range": overlapping_range})
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
"""
|
||||
@@ -388,7 +388,7 @@ class DocumentSplitter:
|
||||
return serialized
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "DocumentSplitter":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "DocumentSplitter":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
"""
|
||||
@@ -402,8 +402,8 @@ class DocumentSplitter:
|
||||
|
||||
@staticmethod
|
||||
def _concatenate_sentences_based_on_word_amount(
|
||||
sentences: List[str], split_length: int, split_overlap: int
|
||||
) -> Tuple[List[str], List[int], List[int]]:
|
||||
sentences: list[str], split_length: int, split_overlap: int
|
||||
) -> tuple[list[str], list[int], list[int]]:
|
||||
"""
|
||||
Groups the sentences into chunks of `split_length` words while respecting sentence boundaries.
|
||||
|
||||
@@ -419,10 +419,10 @@ class DocumentSplitter:
|
||||
chunk_word_count = 0
|
||||
chunk_starting_page_number = 1
|
||||
chunk_start_idx = 0
|
||||
current_chunk: List[str] = []
|
||||
current_chunk: list[str] = []
|
||||
# output lists
|
||||
split_start_page_numbers = []
|
||||
list_of_splits: List[List[str]] = []
|
||||
list_of_splits: list[list[str]] = []
|
||||
split_start_indices = []
|
||||
|
||||
for sentence_idx, sentence in enumerate(sentences):
|
||||
@@ -470,7 +470,7 @@ class DocumentSplitter:
|
||||
return text_splits, split_start_page_numbers, split_start_indices
|
||||
|
||||
@staticmethod
|
||||
def _number_of_sentences_to_keep(sentences: List[str], split_length: int, split_overlap: int) -> int:
|
||||
def _number_of_sentences_to_keep(sentences: list[str], split_length: int, split_overlap: int) -> int:
|
||||
"""
|
||||
Returns the number of sentences to keep in the next chunk based on the `split_overlap` and `split_length`.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any, Dict, List, Literal, Set
|
||||
from typing import Any, Literal
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.components.preprocessors import DocumentSplitter
|
||||
@@ -36,7 +36,7 @@ class HierarchicalDocumentSplitter:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
block_sizes: Set[int],
|
||||
block_sizes: set[int],
|
||||
split_overlap: int = 0,
|
||||
split_by: Literal["word", "sentence", "page", "passage"] = "word",
|
||||
):
|
||||
@@ -49,13 +49,13 @@ class HierarchicalDocumentSplitter:
|
||||
"""
|
||||
|
||||
self.block_sizes = sorted(set(block_sizes), reverse=True)
|
||||
self.splitters: Dict[int, DocumentSplitter] = {}
|
||||
self.splitters: dict[int, DocumentSplitter] = {}
|
||||
self.split_overlap = split_overlap
|
||||
self.split_by = split_by
|
||||
self._build_block_sizes()
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]):
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]):
|
||||
"""
|
||||
Builds a hierarchical document structure for each document in a list of documents.
|
||||
|
||||
@@ -82,7 +82,7 @@ class HierarchicalDocumentSplitter:
|
||||
document.meta["__level"] = 0
|
||||
return document
|
||||
|
||||
def build_hierarchy_from_doc(self, document: Document) -> List[Document]:
|
||||
def build_hierarchy_from_doc(self, document: Document) -> list[Document]:
|
||||
"""
|
||||
Build a hierarchical tree document structure from a single document.
|
||||
|
||||
@@ -119,7 +119,7 @@ class HierarchicalDocumentSplitter:
|
||||
|
||||
return [root] + all_docs
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Returns a dictionary representation of the component.
|
||||
|
||||
@@ -131,7 +131,7 @@ class HierarchicalDocumentSplitter:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HierarchicalDocumentSplitter":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HierarchicalDocumentSplitter":
|
||||
"""
|
||||
Deserialize this component from a dictionary.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from haystack import Document, component, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -62,8 +62,8 @@ class RecursiveDocumentSplitter:
|
||||
split_length: int = 200,
|
||||
split_overlap: int = 0,
|
||||
split_unit: Literal["word", "char", "token"] = "word",
|
||||
separators: Optional[List[str]] = None,
|
||||
sentence_splitter_params: Optional[Dict[str, Any]] = None,
|
||||
separators: Optional[list[str]] = None,
|
||||
sentence_splitter_params: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Initializes a RecursiveDocumentSplitter.
|
||||
@@ -118,12 +118,12 @@ class RecursiveDocumentSplitter:
|
||||
raise ValueError("All separators must be strings.")
|
||||
|
||||
@staticmethod
|
||||
def _get_custom_sentence_tokenizer(sentence_splitter_params: Dict[str, Any]):
|
||||
def _get_custom_sentence_tokenizer(sentence_splitter_params: dict[str, Any]):
|
||||
from haystack.components.preprocessors.sentence_tokenizer import SentenceSplitter
|
||||
|
||||
return SentenceSplitter(**sentence_splitter_params)
|
||||
|
||||
def _split_chunk(self, current_chunk: str) -> Tuple[str, str]:
|
||||
def _split_chunk(self, current_chunk: str) -> tuple[str, str]:
|
||||
"""
|
||||
Splits a chunk based on the split_length and split_units attribute.
|
||||
|
||||
@@ -148,7 +148,7 @@ class RecursiveDocumentSplitter:
|
||||
remaining_tokens = tokens[self.split_length :]
|
||||
return self.tiktoken_tokenizer.decode(current_tokens), self.tiktoken_tokenizer.decode(remaining_tokens) # type: ignore
|
||||
|
||||
def _apply_overlap(self, chunks: List[str]) -> List[str]:
|
||||
def _apply_overlap(self, chunks: list[str]) -> list[str]:
|
||||
"""
|
||||
Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
|
||||
|
||||
@@ -160,7 +160,7 @@ class RecursiveDocumentSplitter:
|
||||
:returns:
|
||||
A list of text chunks with the overlap applied.
|
||||
"""
|
||||
overlapped_chunks: List[str] = []
|
||||
overlapped_chunks: list[str] = []
|
||||
|
||||
for idx, chunk in enumerate(chunks):
|
||||
if idx == 0:
|
||||
@@ -238,7 +238,7 @@ class RecursiveDocumentSplitter:
|
||||
current_chunk = overlap + chunk
|
||||
return current_chunk
|
||||
|
||||
def _get_overlap(self, overlapped_chunks: List[str]) -> Tuple[str, str]:
|
||||
def _get_overlap(self, overlapped_chunks: list[str]) -> tuple[str, str]:
|
||||
"""Get the previous overlapped chunk instead of the original chunk."""
|
||||
prev_chunk = overlapped_chunks[-1]
|
||||
overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
|
||||
@@ -273,7 +273,7 @@ class RecursiveDocumentSplitter:
|
||||
# at this point we know that the tokenizer is already initialized
|
||||
return len(self.tiktoken_tokenizer.encode(text)) # type: ignore
|
||||
|
||||
def _chunk_text(self, text: str) -> List[str]:
|
||||
def _chunk_text(self, text: str) -> list[str]:
|
||||
"""
|
||||
Recursive chunking algorithm that divides text into smaller chunks based on a list of separator characters.
|
||||
|
||||
@@ -313,7 +313,7 @@ class RecursiveDocumentSplitter:
|
||||
continue
|
||||
|
||||
chunks = []
|
||||
current_chunk: List[str] = []
|
||||
current_chunk: list[str] = []
|
||||
current_length = 0
|
||||
|
||||
# check splits, if any is too long, recursively chunk it, otherwise add to current chunk
|
||||
@@ -356,7 +356,7 @@ class RecursiveDocumentSplitter:
|
||||
# if no separator worked, fall back to word- or character-level chunking
|
||||
return self._fall_back_to_fixed_chunking(text, self.split_units)
|
||||
|
||||
def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> List[str]:
|
||||
def _fall_back_to_fixed_chunking(self, text: str, split_units: Literal["word", "char", "token"]) -> list[str]:
|
||||
"""
|
||||
Fall back to a fixed chunking approach if no separator works for the text.
|
||||
|
||||
@@ -399,7 +399,7 @@ class RecursiveDocumentSplitter:
|
||||
chunks.append(self.tiktoken_tokenizer.decode(chunk_tokens)) # type: ignore
|
||||
return chunks
|
||||
|
||||
def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: List[Document]) -> None:
|
||||
def _add_overlap_info(self, curr_pos: int, new_doc: Document, new_docs: list[Document]) -> None:
|
||||
prev_doc = new_docs[-1]
|
||||
overlap_length = self._chunk_length(prev_doc.content) - (curr_pos - prev_doc.meta["split_idx_start"]) # type: ignore
|
||||
if overlap_length > 0:
|
||||
@@ -414,13 +414,13 @@ class RecursiveDocumentSplitter:
|
||||
}
|
||||
)
|
||||
|
||||
def _run_one(self, doc: Document) -> List[Document]:
|
||||
def _run_one(self, doc: Document) -> list[Document]:
|
||||
chunks = self._chunk_text(doc.content) # type: ignore # the caller already check for a non-empty doc.content
|
||||
chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks # remove last empty chunk if it exists
|
||||
current_position = 0
|
||||
current_page = 1
|
||||
|
||||
new_docs: List[Document] = []
|
||||
new_docs: list[Document] = []
|
||||
|
||||
for split_nr, chunk in enumerate(chunks):
|
||||
meta = deepcopy(doc.meta)
|
||||
@@ -452,8 +452,8 @@ class RecursiveDocumentSplitter:
|
||||
|
||||
return new_docs
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Split a list of documents into documents with smaller chunks of text.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Tuple
|
||||
from typing import Any, Literal
|
||||
|
||||
from haystack import logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -144,7 +144,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
|
||||
self.sentence_tokenizer._params.abbrev_types.update(abbreviations)
|
||||
self.keep_white_spaces = keep_white_spaces
|
||||
|
||||
def split_sentences(self, text: str) -> List[Dict[str, Any]]:
|
||||
def split_sentences(self, text: str) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Splits a text into sentences including references to original char positions for each split.
|
||||
|
||||
@@ -159,7 +159,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
|
||||
return sentences
|
||||
|
||||
@staticmethod
|
||||
def _apply_split_rules(text: str, sentence_spans: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
|
||||
def _apply_split_rules(text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
||||
"""
|
||||
Applies additional split rules to the sentence spans.
|
||||
|
||||
@@ -182,7 +182,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
|
||||
|
||||
@staticmethod
|
||||
def _needs_join(
|
||||
text: str, span: Tuple[int, int], next_span: Tuple[int, int], quote_spans: List[Tuple[int, int]]
|
||||
text: str, span: tuple[int, int], next_span: tuple[int, int], quote_spans: list[tuple[int, int]]
|
||||
) -> bool:
|
||||
"""
|
||||
Checks if the spans need to be joined as parts of one sentence.
|
||||
@@ -222,7 +222,7 @@ class SentenceSplitter: # pylint: disable=too-few-public-methods
|
||||
return re.search(r"^\s*[\(\[]", text[next_start:next_end]) is not None
|
||||
|
||||
@staticmethod
|
||||
def _read_abbreviations(lang: Language) -> List[str]:
|
||||
def _read_abbreviations(lang: Language) -> list[str]:
|
||||
"""
|
||||
Reads the abbreviations for a given language from the abbreviations file.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import re
|
||||
import string
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from haystack import component
|
||||
|
||||
@@ -32,7 +32,7 @@ class TextCleaner:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
remove_regexps: Optional[List[str]] = None,
|
||||
remove_regexps: Optional[list[str]] = None,
|
||||
convert_to_lowercase: bool = False,
|
||||
remove_punctuation: bool = False,
|
||||
remove_numbers: bool = False,
|
||||
@@ -61,8 +61,8 @@ class TextCleaner:
|
||||
|
||||
self._translator = str.maketrans("", "", to_remove) if to_remove else None
|
||||
|
||||
@component.output_types(texts=List[str])
|
||||
def run(self, texts: List[str]) -> Dict[str, Any]:
|
||||
@component.output_types(texts=list[str])
|
||||
def run(self, texts: list[str]) -> dict[str, Any]:
|
||||
"""
|
||||
Cleans up the given list of strings.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import copy
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
@@ -66,7 +66,7 @@ class HuggingFaceTEIRanker:
|
||||
raw_scores: bool = False,
|
||||
timeout: Optional[int] = 30,
|
||||
max_retries: int = 3,
|
||||
retry_status_codes: Optional[List[int]] = None,
|
||||
retry_status_codes: Optional[list[int]] = None,
|
||||
token: Optional[Secret] = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
|
||||
) -> None:
|
||||
"""
|
||||
@@ -91,7 +91,7 @@ class HuggingFaceTEIRanker:
|
||||
self.retry_status_codes = retry_status_codes
|
||||
self.raw_scores = raw_scores
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -109,7 +109,7 @@ class HuggingFaceTEIRanker:
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "HuggingFaceTEIRanker":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "HuggingFaceTEIRanker":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -122,8 +122,8 @@ class HuggingFaceTEIRanker:
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _compose_response(
|
||||
self, result: Union[Dict[str, str], List[Dict[str, Any]]], top_k: Optional[int], documents: List[Document]
|
||||
) -> Dict[str, List[Document]]:
|
||||
self, result: Union[dict[str, str], list[dict[str, Any]]], top_k: Optional[int], documents: list[Document]
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Processes the API response into a structured format.
|
||||
|
||||
@@ -162,14 +162,14 @@ class HuggingFaceTEIRanker:
|
||||
ranked_docs.append(doc_copy)
|
||||
return {"documents": ranked_docs}
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
truncation_direction: Optional[TruncationDirection] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Reranks the provided documents by relevance to the query using the TEI API.
|
||||
|
||||
@@ -193,7 +193,7 @@ class HuggingFaceTEIRanker:
|
||||
|
||||
# Prepare the payload
|
||||
texts = [doc.content for doc in documents]
|
||||
payload: Dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
|
||||
payload: dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
|
||||
if truncation_direction:
|
||||
payload.update({"truncate": True, "truncation_direction": truncation_direction.value})
|
||||
|
||||
@@ -212,18 +212,18 @@ class HuggingFaceTEIRanker:
|
||||
status_codes_to_retry=self.retry_status_codes,
|
||||
)
|
||||
|
||||
result: Union[Dict[str, str], List[Dict[str, Any]]] = response.json()
|
||||
result: Union[dict[str, str], list[dict[str, Any]]] = response.json()
|
||||
|
||||
return self._compose_response(result, top_k, documents)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
async def run_async(
|
||||
self,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
truncation_direction: Optional[TruncationDirection] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Asynchronously reranks the provided documents by relevance to the query using the TEI API.
|
||||
|
||||
@@ -246,7 +246,7 @@ class HuggingFaceTEIRanker:
|
||||
|
||||
# Prepare the payload
|
||||
texts = [doc.content for doc in documents]
|
||||
payload: Dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
|
||||
payload: dict[str, Any] = {"query": query, "texts": texts, "raw_scores": self.raw_scores}
|
||||
if truncation_direction:
|
||||
payload.update({"truncate": True, "truncation_direction": truncation_direction.value})
|
||||
|
||||
@@ -265,6 +265,6 @@ class HuggingFaceTEIRanker:
|
||||
status_codes_to_retry=self.retry_status_codes,
|
||||
)
|
||||
|
||||
result: Union[Dict[str, str], List[Dict[str, Any]]] = response.json()
|
||||
result: Union[dict[str, str], list[dict[str, Any]]] = response.json()
|
||||
|
||||
return self._compose_response(result, top_k, documents)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from haystack import Document, component
|
||||
|
||||
@@ -59,10 +59,10 @@ class LostInTheMiddleRanker:
|
||||
self.word_count_threshold = word_count_threshold
|
||||
self.top_k = top_k
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self, documents: List[Document], top_k: Optional[int] = None, word_count_threshold: Optional[int] = None
|
||||
) -> Dict[str, List[Document]]:
|
||||
self, documents: list[Document], top_k: Optional[int] = None, word_count_threshold: Optional[int] = None
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Reranks documents based on the "lost in the middle" order.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional
|
||||
from typing import Any, Callable, Literal, Optional
|
||||
|
||||
from dateutil.parser import parse as date_parse
|
||||
|
||||
@@ -156,10 +156,10 @@ class MetaFieldRanker:
|
||||
"MetaFieldRanker." % meta_value_type
|
||||
)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
weight: Optional[float] = None,
|
||||
ranking_mode: Optional[Literal["reciprocal_rank_fusion", "linear_score"]] = None,
|
||||
@@ -320,8 +320,8 @@ class MetaFieldRanker:
|
||||
return {"documents": sorted_documents[:top_k]}
|
||||
|
||||
def _parse_meta(
|
||||
self, docs_with_meta_field: List[Document], meta_value_type: Optional[Literal["float", "int", "date"]]
|
||||
) -> List[Any]:
|
||||
self, docs_with_meta_field: list[Document], meta_value_type: Optional[Literal["float", "int", "date"]]
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Parse the meta values stored under `self.meta_field` for the Documents provided in `docs_with_meta_field`.
|
||||
"""
|
||||
@@ -364,15 +364,15 @@ class MetaFieldRanker:
|
||||
|
||||
def _merge_rankings(
|
||||
self,
|
||||
documents: List[Document],
|
||||
sorted_documents: List[Document],
|
||||
documents: list[Document],
|
||||
sorted_documents: list[Document],
|
||||
weight: float,
|
||||
ranking_mode: Literal["reciprocal_rank_fusion", "linear_score"],
|
||||
) -> List[Document]:
|
||||
) -> list[Document]:
|
||||
"""
|
||||
Merge the two different rankings for Documents sorted both by their content and by their meta field.
|
||||
"""
|
||||
scores_map: Dict = defaultdict(int)
|
||||
scores_map: dict = defaultdict(int)
|
||||
|
||||
if ranking_mode == "reciprocal_rank_fusion":
|
||||
for i, (document, sorted_doc) in enumerate(zip(documents, sorted_documents)):
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List, Optional, cast
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from haystack import Document, component
|
||||
|
||||
@@ -72,8 +72,8 @@ class MetaFieldGroupingRanker:
|
||||
self.sort_docs_by = sort_docs_by
|
||||
self.subgroup_by = subgroup_by
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
def run(self, documents: List[Document]) -> Dict[str, Any]:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, documents: list[Document]) -> dict[str, Any]:
|
||||
"""
|
||||
Groups the provided list of documents based on the `group_by` parameter and optionally the `subgroup_by`.
|
||||
|
||||
@@ -88,7 +88,7 @@ class MetaFieldGroupingRanker:
|
||||
if not documents:
|
||||
return {"documents": []}
|
||||
|
||||
document_groups: Dict[str, Dict[str, List[Document]]] = defaultdict(lambda: defaultdict(list))
|
||||
document_groups: dict[str, dict[str, list[Document]]] = defaultdict(lambda: defaultdict(list))
|
||||
no_group_docs = []
|
||||
|
||||
for doc in documents:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -120,13 +120,13 @@ class SentenceTransformersDiversityRanker:
|
||||
query_suffix: str = "",
|
||||
document_prefix: str = "",
|
||||
document_suffix: str = "",
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
strategy: Union[str, DiversityRankingStrategy] = "greedy_diversity_order",
|
||||
lambda_threshold: float = 0.5,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
):
|
||||
"""
|
||||
@@ -206,7 +206,7 @@ class SentenceTransformersDiversityRanker:
|
||||
backend=self.backend,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -238,7 +238,7 @@ class SentenceTransformersDiversityRanker:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersDiversityRanker":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersDiversityRanker":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -255,7 +255,7 @@ class SentenceTransformersDiversityRanker:
|
||||
deserialize_hf_model_kwargs(init_params["model_kwargs"])
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
def _prepare_texts_to_embed(self, documents: List[Document]) -> List[str]:
|
||||
def _prepare_texts_to_embed(self, documents: list[Document]) -> list[str]:
|
||||
"""
|
||||
Prepare the texts to embed by concatenating the Document text with the metadata fields to embed.
|
||||
"""
|
||||
@@ -273,7 +273,7 @@ class SentenceTransformersDiversityRanker:
|
||||
|
||||
return texts_to_embed
|
||||
|
||||
def _greedy_diversity_order(self, query: str, documents: List[Document]) -> List[Document]:
|
||||
def _greedy_diversity_order(self, query: str, documents: list[Document]) -> list[Document]:
|
||||
"""
|
||||
Orders the given list of documents to maximize diversity.
|
||||
|
||||
@@ -293,7 +293,7 @@ class SentenceTransformersDiversityRanker:
|
||||
doc_embeddings, query_embedding = self._embed_and_normalize(query, texts_to_embed)
|
||||
|
||||
n = len(documents)
|
||||
selected: List[int] = []
|
||||
selected: list[int] = []
|
||||
|
||||
# Compute the similarity vector between the query and documents
|
||||
query_doc_sim = query_embedding @ doc_embeddings.T
|
||||
@@ -315,7 +315,7 @@ class SentenceTransformersDiversityRanker:
|
||||
# It's divided by n for numerical stability
|
||||
selected_sum += doc_embeddings[index_unselected] / n
|
||||
|
||||
ranked_docs: List[Document] = [documents[i] for i in selected]
|
||||
ranked_docs: list[Document] = [documents[i] for i in selected]
|
||||
|
||||
return ranked_docs
|
||||
|
||||
@@ -333,8 +333,8 @@ class SentenceTransformersDiversityRanker:
|
||||
return doc_embeddings, query_embedding
|
||||
|
||||
def _maximum_margin_relevance(
|
||||
self, query: str, documents: List[Document], lambda_threshold: float, top_k: int
|
||||
) -> List[Document]:
|
||||
self, query: str, documents: list[Document], lambda_threshold: float, top_k: int
|
||||
) -> list[Document]:
|
||||
"""
|
||||
Orders the given list of documents according to the Maximum Margin Relevance (MMR) scores.
|
||||
|
||||
@@ -355,7 +355,7 @@ class SentenceTransformersDiversityRanker:
|
||||
doc_embeddings, query_embedding = self._embed_and_normalize(query, texts_to_embed)
|
||||
top_k = top_k if top_k else len(documents)
|
||||
|
||||
selected: List[int] = []
|
||||
selected: list[int] = []
|
||||
query_similarities_as_tensor = query_embedding @ doc_embeddings.T
|
||||
query_similarities = query_similarities_as_tensor.reshape(-1)
|
||||
idx = int(torch.argmax(query_similarities))
|
||||
@@ -386,14 +386,14 @@ class SentenceTransformersDiversityRanker:
|
||||
if (strategy == DiversityRankingStrategy.MAXIMUM_MARGIN_RELEVANCE) and not 0 <= lambda_threshold <= 1:
|
||||
raise ValueError(f"lambda_threshold must be between 0 and 1, but got {lambda_threshold}.")
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
lambda_threshold: Optional[float] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Rank the documents based on their diversity.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from copy import copy
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
from typing import Any, Literal, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -48,14 +48,14 @@ class SentenceTransformersSimilarityRanker:
|
||||
top_k: int = 10,
|
||||
query_prefix: str = "",
|
||||
document_prefix: str = "",
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
scale_score: bool = True,
|
||||
score_threshold: Optional[float] = None,
|
||||
trust_remote_code: bool = False,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
config_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
config_kwargs: Optional[dict[str, Any]] = None,
|
||||
backend: Literal["torch", "onnx", "openvino"] = "torch",
|
||||
batch_size: int = 16,
|
||||
):
|
||||
@@ -130,7 +130,7 @@ class SentenceTransformersSimilarityRanker:
|
||||
self.backend = backend
|
||||
self.batch_size = batch_size
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
@@ -152,7 +152,7 @@ class SentenceTransformersSimilarityRanker:
|
||||
backend=self.backend,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -183,7 +183,7 @@ class SentenceTransformersSimilarityRanker:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SentenceTransformersSimilarityRanker":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "SentenceTransformersSimilarityRanker":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -201,16 +201,16 @@ class SentenceTransformersSimilarityRanker:
|
||||
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
scale_score: Optional[bool] = None,
|
||||
score_threshold: Optional[float] = None,
|
||||
) -> Dict[str, List[Document]]:
|
||||
) -> dict[str, list[Document]]:
|
||||
"""
|
||||
Returns a list of documents ranked by their similarity to the given query.
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -56,13 +56,13 @@ class TransformersSimilarityRanker:
|
||||
top_k: int = 10,
|
||||
query_prefix: str = "",
|
||||
document_prefix: str = "",
|
||||
meta_fields_to_embed: Optional[List[str]] = None,
|
||||
meta_fields_to_embed: Optional[list[str]] = None,
|
||||
embedding_separator: str = "\n",
|
||||
scale_score: bool = True,
|
||||
calibration_factor: Optional[float] = 1.0,
|
||||
score_threshold: Optional[float] = None,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenizer_kwargs: Optional[dict[str, Any]] = None,
|
||||
batch_size: int = 16,
|
||||
):
|
||||
"""
|
||||
@@ -146,7 +146,7 @@ class TransformersSimilarityRanker:
|
||||
if self.top_k <= 0:
|
||||
raise ValueError(f"top_k must be > 0, but got {top_k}")
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
@@ -168,7 +168,7 @@ class TransformersSimilarityRanker:
|
||||
assert self.model is not None
|
||||
self.device = ComponentDevice.from_multiple(device_map=DeviceMap.from_hf(self.model.hf_device_map))
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -197,7 +197,7 @@ class TransformersSimilarityRanker:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "TransformersSimilarityRanker":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "TransformersSimilarityRanker":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -215,11 +215,11 @@ class TransformersSimilarityRanker:
|
||||
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
@component.output_types(documents=List[Document])
|
||||
@component.output_types(documents=list[Document])
|
||||
def run( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
scale_score: Optional[bool] = None,
|
||||
calibration_factor: Optional[float] = None,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from haystack import Document, ExtractedAnswer, component, default_from_dict, default_to_dict, logging
|
||||
from haystack.lazy_imports import LazyImport
|
||||
@@ -64,7 +64,7 @@ class ExtractiveReader:
|
||||
no_answer: bool = True,
|
||||
calibration_factor: float = 0.1,
|
||||
overlap_threshold: Optional[float] = 0.01,
|
||||
model_kwargs: Optional[Dict[str, Any]] = None,
|
||||
model_kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Creates an instance of ExtractiveReader.
|
||||
@@ -126,13 +126,13 @@ class ExtractiveReader:
|
||||
model_kwargs = resolve_hf_device_map(device=device, model_kwargs=model_kwargs)
|
||||
self.model_kwargs = model_kwargs
|
||||
|
||||
def _get_telemetry_data(self) -> Dict[str, Any]:
|
||||
def _get_telemetry_data(self) -> dict[str, Any]:
|
||||
"""
|
||||
Data that is sent to Posthog for usage analytics.
|
||||
"""
|
||||
return {"model": self.model_name_or_path}
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
Serializes the component to a dictionary.
|
||||
|
||||
@@ -159,7 +159,7 @@ class ExtractiveReader:
|
||||
return serialization_dict
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ExtractiveReader":
|
||||
def from_dict(cls, data: dict[str, Any]) -> "ExtractiveReader":
|
||||
"""
|
||||
Deserializes the component from a dictionary.
|
||||
|
||||
@@ -194,8 +194,8 @@ class ExtractiveReader:
|
||||
|
||||
@staticmethod
|
||||
def _flatten_documents(
|
||||
queries: List[str], documents: List[List[Document]]
|
||||
) -> Tuple[List[str], List[Document], List[int]]:
|
||||
queries: list[str], documents: list[list[Document]]
|
||||
) -> tuple[list[str], list[Document], list[int]]:
|
||||
"""
|
||||
Flattens queries and Documents so all query-document pairs are arranged along one batch axis.
|
||||
"""
|
||||
@@ -205,8 +205,8 @@ class ExtractiveReader:
|
||||
return flattened_queries, flattened_documents, query_ids
|
||||
|
||||
def _preprocess( # pylint: disable=too-many-positional-arguments
|
||||
self, *, queries: List[str], documents: List[Document], max_seq_length: int, query_ids: List[int], stride: int
|
||||
) -> Tuple["torch.Tensor", "torch.Tensor", "torch.Tensor", List["Encoding"], List[int], List[int]]:
|
||||
self, *, queries: list[str], documents: list[Document], max_seq_length: int, query_ids: list[int], stride: int
|
||||
) -> tuple["torch.Tensor", "torch.Tensor", "torch.Tensor", list["Encoding"], list[int], list[int]]:
|
||||
"""
|
||||
Splits and tokenizes Documents and preserves structures by returning mappings to query and Document IDs.
|
||||
"""
|
||||
@@ -264,8 +264,8 @@ class ExtractiveReader:
|
||||
sequence_ids: "torch.Tensor",
|
||||
attention_mask: "torch.Tensor",
|
||||
answers_per_seq: int,
|
||||
encodings: List["Encoding"],
|
||||
) -> Tuple[List[List[int]], List[List[int]], "torch.Tensor"]:
|
||||
encodings: list["Encoding"],
|
||||
) -> tuple[list[list[int]], list[list[int]], "torch.Tensor"]:
|
||||
"""
|
||||
Turns start and end logits into probabilities for each answer span.
|
||||
|
||||
@@ -347,19 +347,19 @@ class ExtractiveReader:
|
||||
def _nest_answers(
|
||||
self,
|
||||
*,
|
||||
start: List[List[int]],
|
||||
end: List[List[int]],
|
||||
start: list[list[int]],
|
||||
end: list[list[int]],
|
||||
probabilities: "torch.Tensor",
|
||||
flattened_documents: List[Document],
|
||||
queries: List[str],
|
||||
flattened_documents: list[Document],
|
||||
queries: list[str],
|
||||
answers_per_seq: int,
|
||||
top_k: Optional[int],
|
||||
score_threshold: Optional[float],
|
||||
query_ids: List[int],
|
||||
document_ids: List[int],
|
||||
query_ids: list[int],
|
||||
document_ids: list[int],
|
||||
no_answer: bool,
|
||||
overlap_threshold: Optional[float],
|
||||
) -> List[List[ExtractedAnswer]]:
|
||||
) -> list[list[ExtractedAnswer]]:
|
||||
"""
|
||||
Reconstructs the nested structure that existed before flattening.
|
||||
|
||||
@@ -431,7 +431,7 @@ class ExtractiveReader:
|
||||
return 0
|
||||
|
||||
def _should_keep(
|
||||
self, candidate_answer: ExtractedAnswer, current_answers: List[ExtractedAnswer], overlap_threshold: float
|
||||
self, candidate_answer: ExtractedAnswer, current_answers: list[ExtractedAnswer], overlap_threshold: float
|
||||
) -> bool:
|
||||
"""
|
||||
Determines if the answer should be kept based on how much it overlaps with previous answers.
|
||||
@@ -493,8 +493,8 @@ class ExtractiveReader:
|
||||
return keep
|
||||
|
||||
def deduplicate_by_overlap(
|
||||
self, answers: List[ExtractedAnswer], overlap_threshold: Optional[float]
|
||||
) -> List[ExtractedAnswer]:
|
||||
self, answers: list[ExtractedAnswer], overlap_threshold: Optional[float]
|
||||
) -> list[ExtractedAnswer]:
|
||||
"""
|
||||
De-duplicates overlapping Extractive Answers.
|
||||
|
||||
@@ -529,11 +529,11 @@ class ExtractiveReader:
|
||||
|
||||
return deduplicated_answers
|
||||
|
||||
@component.output_types(answers=List[ExtractedAnswer])
|
||||
@component.output_types(answers=list[ExtractedAnswer])
|
||||
def run( # pylint: disable=too-many-positional-arguments
|
||||
self,
|
||||
query: str,
|
||||
documents: List[Document],
|
||||
documents: list[Document],
|
||||
top_k: Optional[int] = None,
|
||||
score_threshold: Optional[float] = None,
|
||||
max_seq_length: Optional[int] = None,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user