mirror of
https://github.com/microsoft/semantic-kernel.git
synced 2026-10-12 11:19:56 +00:00
Eduard van Valkenburg
003cc4c010
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Overhaul of the VectorStores to get it matching with dotnet and the latest development there. Closes: - #11938 - #11598 - #11597 - #11517 - #10561 - #10391 - #9911 - #9892 - #10867 ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Many changes, some highlights: - Moved from different vector fields types, to a single with a enum first VectorStoreField() - Renamed several items - Made embeddinggenerrator a part of the vector field spec and do automatic vectorization from there or from the same param on a collection - Adds hybrid search using the same setup - Removed the intermediate TextSeachVectorSearch object need, by allowing `create_search_function` directly on a collection Will write out full changes in a migration guide in the docs. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
73 lines
2.5 KiB
Python
73 lines
2.5 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
|
|
import asyncio
|
|
from uuid import uuid4
|
|
|
|
import pandas as pd
|
|
|
|
from semantic_kernel.connectors.ai.open_ai import OpenAITextEmbedding
|
|
from semantic_kernel.connectors.azure_ai_search import AzureAISearchCollection
|
|
from semantic_kernel.data.vector import VectorStoreCollectionDefinition, VectorStoreField
|
|
|
|
definition = VectorStoreCollectionDefinition(
|
|
collection_name="pandas_test_index",
|
|
fields=[
|
|
VectorStoreField("key", name="id", type="str"),
|
|
VectorStoreField("data", name="title", type="str"),
|
|
VectorStoreField("data", name="content", type="str", is_full_text_indexed=True),
|
|
VectorStoreField(
|
|
"vector",
|
|
name="vector",
|
|
type="float",
|
|
dimensions=1536,
|
|
embedding_generator=OpenAITextEmbedding(ai_model_id="text-embedding-3-small"),
|
|
),
|
|
],
|
|
to_dict=lambda record, **_: record.to_dict(orient="records"),
|
|
from_dict=lambda records, **_: pd.DataFrame(records),
|
|
container_mode=True,
|
|
)
|
|
|
|
|
|
async def main():
|
|
# create the record collection
|
|
async with AzureAISearchCollection[str, pd.DataFrame](
|
|
record_type=pd.DataFrame,
|
|
definition=definition,
|
|
) as collection:
|
|
await collection.ensure_collection_exists()
|
|
# create some records
|
|
records = [
|
|
{
|
|
"id": str(uuid4()),
|
|
"title": "Document about Semantic Kernel.",
|
|
"content": "Semantic Kernel is a framework for building AI applications.",
|
|
},
|
|
{
|
|
"id": str(uuid4()),
|
|
"title": "Document about Python",
|
|
"content": "Python is a programming language that lets you work quickly.",
|
|
},
|
|
]
|
|
|
|
# create the dataframe and add the content you want to embed to a new column
|
|
df = pd.DataFrame(records)
|
|
df["vector"] = df.apply(lambda row: f"title: {row['title']}, content: {row['content']}", axis=1)
|
|
print(df.head(1))
|
|
# upsert the records (for a container, upsert and upsert_batch are equivalent)
|
|
await collection.upsert(df)
|
|
|
|
# retrieve a record
|
|
result = await collection.get(top=2)
|
|
if result is None:
|
|
print("No records found, this is sometimes because the get is too fast and the index is not ready yet.")
|
|
else:
|
|
print("Retrieved records:")
|
|
print(result.to_string())
|
|
|
|
await collection.ensure_collection_deleted()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|