microsoft/semantic-kernel
1
0
Fork 0
mirror of https://github.com/microsoft/semantic-kernel.git synced 2026-10-12 02:19:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Java: 2636 net java azurecognitivesearchmemory should implement memorystore (#2861)

Browse Source 
### Motivation and Context
Fixes issue #2636 

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄
This commit is contained in:
David Grieve
2023-09-19 17:37:25 -04:00
committed by GitHub
parent bb0d256229
commit 0ede0eb534
13 changed files with 792 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/pom.xml
+25 -1
View File
@@ -20,11 +20,35 @@
<groupId>com.microsoft.semantic-kernel</groupId>
<artifactId>semantickernel-api</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-ai-openai</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-search-documents</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemory.java
-401
View File
@@ -1,401 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.connectors.memory.azurecognitivesearch;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.credential.TokenCredential;
import com.azure.core.http.rest.Response;
import com.azure.search.documents.SearchAsyncClient;
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
import com.azure.search.documents.indexes.models.PrioritizedFields;
import com.azure.search.documents.indexes.models.SearchField;
import com.azure.search.documents.indexes.models.SearchIndex;
import com.azure.search.documents.indexes.models.SemanticConfiguration;
import com.azure.search.documents.indexes.models.SemanticField;
import com.azure.search.documents.indexes.models.SemanticSettings;
import com.azure.search.documents.models.IndexDocumentsOptions;
import com.azure.search.documents.models.IndexDocumentsResult;
import com.azure.search.documents.models.QueryLanguage;
import com.azure.search.documents.models.QueryType;
import com.azure.search.documents.models.SearchOptions;
import com.azure.search.documents.util.SearchPagedResponse;
import com.microsoft.semantickernel.memory.MemoryException;
import com.microsoft.semantickernel.memory.MemoryException.ErrorCodes;
import com.microsoft.semantickernel.memory.MemoryQueryResult;
import com.microsoft.semantickernel.memory.MemoryRecordMetadata;
import com.microsoft.semantickernel.memory.SemanticTextMemory;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import reactor.core.publisher.Mono;
/**
* Semantic Memory implementation using Azure Cognitive Search. For more information about Azure
* Cognitive Search {@see https://learn.microsoft.com/azure/search/search-what-is-azure-search}
*/
public class AzureCognitiveSearchMemory implements SemanticTextMemory {
private static final String EMPTY_STRING = "";
private final SearchIndexAsyncClient _adminClient;
private final ConcurrentMap<String, SearchAsyncClient> _clientsByIndex =
new ConcurrentHashMap<>();
/**
* Create a new instance of semantic memory using Azure Cognitive Search.
*
* @param endpoint Azure Cognitive Search URI, e.g. {@code "https://contoso.search.windows.net"}
* @param apiKey The key used to authenticate HTTP requests sent to the Azure Cognitive Search
* service.
*/
public AzureCognitiveSearchMemory(String endpoint, String apiKey) {
AzureKeyCredential credentials = new AzureKeyCredential(apiKey);
this._adminClient =
new SearchIndexClientBuilder()
.endpoint(endpoint)
.credential(credentials)
.buildAsyncClient();
}
/**
* Create a new instance of semantic memory using Azure Cognitive Search.
*
* @param endpoint Azure Cognitive Search URI, e.g. {@code "https://contoso.search.windows.net"}
* @param credentials TokenCredential used to authorize requests sent to the Azure Cognitive
* Search service.
*/
public AzureCognitiveSearchMemory(String endpoint, TokenCredential credentials) {
this._adminClient =
new SearchIndexClientBuilder()
.endpoint(endpoint)
.credential(credentials)
.buildAsyncClient();
}
AzureCognitiveSearchMemory(SearchIndexAsyncClient adminClient) {
this._adminClient = adminClient;
}
@Override
public AzureCognitiveSearchMemory copy() {
return new AzureCognitiveSearchMemory(this._adminClient);
}
/**
* @throws IllegalArgumentException if the collection name does not conform to Azure Cognitive
* Search <a
* href="https://learn.microsoft.com/en-us/rest/api/searchservice/naming-rules#chart-of-naming-rules">Index
* naming rules</a>. The name must be lower-case, start with a letter or number, have no
* slashes or dots, and be 2 to 128 characters in length. After starting the name with a
* letter or number, the rest of the name can include any letter, number and dashes, as long
* as the dashes aren't consecutive.
*/
@Override
public Mono<String> saveInformationAsync(
@Nonnull String collection,
@Nonnull String text,
@Nonnull String id,
@Nullable String description,
@Nullable String additionalMetadata) {
collection = normalizeIndexName(collection);
AzureCognitiveSearchRecord record =
new AzureCognitiveSearchRecord(
encodeId(id), text, description, additionalMetadata, null, false);
return this.upsertRecordAsync(collection, record);
}
/**
* @throws IllegalArgumentException if the collection name does not conform to Azure Cognitive
* Search <a
* href="https://learn.microsoft.com/en-us/rest/api/searchservice/naming-rules#chart-of-naming-rules">Index
* naming rules</a>. The name must be lower-case, start with a letter or number, have no
* slashes or dots, and be 2 to 128 characters in length. After starting the name with a
* letter or number, the rest of the name can include any letter, number and dashes, as long
* as the dashes aren't consecutive.
*/
@Override
public Mono<String> saveReferenceAsync(
@Nonnull String collection,
@Nonnull String text,
@Nonnull String externalId,
@Nonnull String externalSourceName,
@Nullable String description,
@Nullable String additionalMetadata) {
collection = normalizeIndexName(collection);
AzureCognitiveSearchRecord record =
new AzureCognitiveSearchRecord(
encodeId(externalId),
text,
description,
additionalMetadata,
externalSourceName,
true);
return this.upsertRecordAsync(collection, record);
}
/**
* @throws IllegalArgumentException if the collection name does not conform to Azure Cognitive
* Search <a
* href="https://learn.microsoft.com/en-us/rest/api/searchservice/naming-rules#chart-of-naming-rules">Index
* naming rules</a>. The name must be lower-case, start with a letter or number, have no
* slashes or dots, and be 2 to 128 characters in length. After starting the name with a
* letter or number, the rest of the name can include any letter, number and dashes, as long
* as the dashes aren't consecutive.
*/
@Override
public Mono<MemoryQueryResult> getAsync(
@Nonnull String collection, @Nonnull String key, boolean withEmbedding) {
SearchAsyncClient client = this.getSearchClient(normalizeIndexName(collection));
return client.getDocumentWithResponse(encodeId(key), AzureCognitiveSearchRecord.class, null)
.map(
(response) -> {
if (response.getStatusCode() == 404 || response.getValue() == null) {
throw new MemoryException(
ErrorCodes.MEMORY_NOT_FOUND, "Memory read returned null");
}
return new MemoryQueryResult(
toMemoryRecordMetadata(response.getValue()), 1);
});
}
/**
* @throws IllegalArgumentException if the collection name does not conform to Azure Cognitive
* Search <a
* href="https://learn.microsoft.com/en-us/rest/api/searchservice/naming-rules#chart-of-naming-rules">Index
* naming rules</a>. The name must be lower-case, start with a letter or number, have no
* slashes or dots, and be 2 to 128 characters in length. After starting the name with a
* letter or number, the rest of the name can include any letter, number and dashes, as long
* as the dashes aren't consecutive.
*/
@Override
public Mono<Void> removeAsync(@Nonnull String collection, @Nonnull String key) {
SearchAsyncClient client = this.getSearchClient(normalizeIndexName(collection));
List<AzureCognitiveSearchRecord> records =
Collections.singletonList(
new AzureCognitiveSearchRecord(encodeId(key), "", "", "", "", false));
return client.deleteDocumentsWithResponse(records, null).then();
}
private Collector<SearchPagedResponse, ?, List<MemoryQueryResult>> toMemoryQueryResultList =
Collector.of(
ArrayList::new,
(list, response) -> {
if (response.getStatusCode() == 404 || response.getValue() == null) {
throw new MemoryException(
ErrorCodes.MEMORY_NOT_FOUND, "Memory read returned null");
}
response.getValue().stream()
.map(
searchResult ->
new MemoryQueryResult(
toMemoryRecordMetadata(
searchResult.getDocument(
AzureCognitiveSearchRecord
.class)),
searchResult.getScore()))
.forEach(list::add);
},
(left, right) -> {
left.addAll(right);
return left;
});
/**
* @throws IllegalArgumentException if the collection name does not conform to Azure Cognitive
* Search <a
* href="https://learn.microsoft.com/en-us/rest/api/searchservice/naming-rules#chart-of-naming-rules">Index
* naming rules</a>. The name must be lower-case, start with a letter or number, have no
* slashes or dots, and be 2 to 128 characters in length. After starting the name with a
* letter or number, the rest of the name can include any letter, number and dashes, as long
* as the dashes aren't consecutive.
*/
@Override
public Mono<List<MemoryQueryResult>> searchAsync(
@Nonnull String collection,
@Nonnull String query,
int limit,
float minRelevanceScore,
boolean withEmbeddings) {
SearchAsyncClient client = this.getSearchClient(normalizeIndexName(collection));
SearchOptions options =
new SearchOptions()
.setQueryType(QueryType.SEMANTIC)
.setSemanticConfigurationName("default")
.setQueryLanguage(QueryLanguage.EN_US)
.setTop(limit);
return client.search(query, options).byPage().collect(toMemoryQueryResultList);
}
@Override
public Mono<List<String>> getCollectionsAsync() {
return this._adminClient
.listIndexes()
.map(SearchIndex::getName)
.collect(Collectors.toList());
}
// Index names cannot contain special chars. We use this rule to replace a few common ones
// with an underscore and reduce the chance of errors. If other special chars are used, we leave
// it
// to the service to throw an error.
// Note:
// - replacing chars introduces a small chance of conflicts, e.g. "the-user" and "the_user".
// - we should consider whether making this optional and leave it to the developer to handle.
//
// Java 8 does not support \s for whitespace, so we have to list them all.
private static String whitespace = " |\\t|\\n|\\x0B|\\f|\\r";
private static String s_replaceIndexNameSymbolsRegex = "[" + whitespace + "|\\|/|.|_|:]";
// Get a search client for the index specified.
// Note: the index might not exist, but we avoid checking everytime and the extra latency.
// <param name="indexName">Index name</param>
// <returns>Search client ready to read/write</returns>
private SearchAsyncClient getSearchClient(String indexName) {
// Search an available client from the local cache
String normalizeIndexName = normalizeIndexName(indexName);
return _clientsByIndex.computeIfAbsent(
indexName, k -> _adminClient.getSearchAsyncClient(normalizeIndexName));
}
// Create a new search index.
// <param name="indexName">Index name</param>
// <param name="cancellationToken">Task cancellation token</param>
// private Mono<SearchIndex> createIndexAsync(String indexName) {
private Mono<SearchIndex> createIndexAsync(String indexName) {
indexName = normalizeIndexName(indexName);
// TODO: Use FieldBuilder pending resolution of
// https://github.com/Azure/azure-sdk-for-java/issues/35584
// List<SearchField> fields =
// FieldBuilder.build(AzureCognitiveSearchRecord.class, null);
List<SearchField> fields = AzureCognitiveSearchRecord.searchFields();
SemanticField titleField = new SemanticField().setFieldName("Description");
List<SemanticField> prioritizedContentFields =
Arrays.asList(
new SemanticField().setFieldName("Text"),
new SemanticField().setFieldName("AdditionalMetadata"));
PrioritizedFields prioritizedFields =
new PrioritizedFields()
.setTitleField(titleField)
.setPrioritizedContentFields(prioritizedContentFields);
SemanticConfiguration semanticConfiguration =
new SemanticConfiguration("default", prioritizedFields);
SemanticSettings semanticSettings =
new SemanticSettings()
.setConfigurations(Collections.singletonList(semanticConfiguration));
SearchIndex newIndex =
new SearchIndex(indexName, fields).setSemanticSettings(semanticSettings);
return this._adminClient.createIndex(newIndex);
}
@SuppressWarnings("unchecked")
private Mono<String> upsertRecordAsync(String indexName, AzureCognitiveSearchRecord record) {
SearchAsyncClient client = this.getSearchClient(indexName);
IndexDocumentsOptions throwOnAnyError =
new IndexDocumentsOptions().setThrowOnAnyError(true);
Function<Response<IndexDocumentsResult>, String> transform =
(response) -> {
if (response.getStatusCode() == 404
|| response.getValue() == null
|| response.getValue().getResults() == null
|| response.getValue().getResults().isEmpty()) {
throw new MemoryException(
ErrorCodes.MEMORY_NOT_FOUND,
"Memory write returned null or an empty set");
}
return response.getValue().getResults().get(0).getKey();
};
return client.mergeOrUploadDocumentsWithResponse(
Collections.singleton(record), throwOnAnyError)
.map(transform)
.onErrorResume(
e ->
createIndexAsync(indexName)
.then(
client.mergeOrUploadDocumentsWithResponse(
Collections.singleton(record),
throwOnAnyError))
.map(transform));
}
private static MemoryRecordMetadata toMemoryRecordMetadata(AzureCognitiveSearchRecord data) {
return new MemoryRecordMetadata(
data.isReference(),
decodeId(data.getId()),
normalizeNullToEmptyString(data.getText()),
normalizeNullToEmptyString(data.getDescription()),
normalizeNullToEmptyString(data.getExternalSourceName()),
normalizeNullToEmptyString(data.getAdditionalMetadata()));
}
private static String normalizeNullToEmptyString(@Nullable String value) {
return value != null ? value : EMPTY_STRING;
}
// Normalize index name to match ACS rules.
// See https://learn.microsoft.com/en-us/rest/api/searchservice/Create-Index
// The method doesn't handle all the error scenarios, leaving it to the service
// to throw an error for edge cases not handled locally.
// <param name="indexName">Value to normalize</param>
// <returns>Normalized name</returns>
private static String normalizeIndexName(String indexName) {
if (indexName.length() > 128) {
throw new IllegalArgumentException("The collection name cannot exceed 128 chars");
}
indexName = indexName.toLowerCase(Locale.ROOT);
return indexName.replaceAll(s_replaceIndexNameSymbolsRegex, "-");
}
// ACS keys can contain only letters, digits, underscore, dash, equal sign, recommending
// to encode values with a URL-safe algorithm.
// <param name="realId">Original Id</param>
// <returns>Encoded id</returns>
private static String encodeId(@Nullable String realId) {
if (realId == null) {
return EMPTY_STRING;
}
byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8));
return new String(bytes, StandardCharsets.UTF_8);
}
private static String decodeId(@Nullable String encodedId) {
if (encodedId == null) {
return EMPTY_STRING;
}
byte[] bytes = Base64.getUrlDecoder().decode(encodedId.getBytes(StandardCharsets.UTF_8));
return new String(bytes, StandardCharsets.UTF_8);
}
}
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryException.java
+64
View File
@@ -0,0 +1,64 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.connectors.memory.azurecognitivesearch;
import com.microsoft.semantickernel.SKException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class AzureCognitiveSearchMemoryException extends SKException {
private final ErrorCodes errorCode;
/**
* Create an exception with a message
*
* @param errorCode The error code
*/
public AzureCognitiveSearchMemoryException(@Nonnull ErrorCodes errorCode) {
this(errorCode, null, null);
}
/**
* Create an exception with a message
*
* @param message a description of the cause of the exception
*/
public AzureCognitiveSearchMemoryException(
@Nonnull ErrorCodes errorCode, @Nullable String message) {
this(errorCode, message, null);
}
/**
* Create an exception with a message and a cause
*
* @param errorCode the error code
* @param message a description of the cause of the exception
* @param cause the cause of the exception
*/
public AzureCognitiveSearchMemoryException(
@Nonnull ErrorCodes errorCode, @Nullable String message, @Nullable Throwable cause) {
super(message, cause);
this.errorCode = errorCode;
}
public ErrorCodes getErrorCode() {
return errorCode;
}
public enum ErrorCodes {
UNKNOWN_ERROR("Unknown error"),
INVALID_EMBEDDING_SIZE("Invalid embedding size"),
INVALID_INDEX_NAME("Invalid index name"),
READ_FAILURE("Read failure"),
WRITE_FAILURE("Write failure");
final String message;
ErrorCodes(String message) {
this.message = message;
}
public String getMessage() {
return message;
}
}
}
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryRecord.java
+179
View File
@@ -0,0 +1,179 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.connectors.memory.azurecognitivesearch;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.microsoft.semantickernel.ai.embeddings.Embedding;
import com.microsoft.semantickernel.memory.MemoryRecord;
import com.microsoft.semantickernel.memory.MemoryRecordMetadata;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
/**
* Azure Cognitive Search record and index definition. Note: once defined, index cannot be modified.
*/
public final class AzureCognitiveSearchMemoryRecord {
static final String ID = "Id";
static final String TEXT = "Text";
static final String DESCRIPTION = "Description";
static final String ADDITIONAL_METADATA = "AdditionalMetadata";
static final String EMBEDDING = "Embedding";
static final String EXTERNAL_SOURCE_NAME = "ExternalSourceName";
static final String IS_REFERENCE = "Reference";
private final String id;
private final String text;
private final String description;
private final String additionalMetadata;
private final List<Float> embedding;
private final String externalSourceName;
private final boolean isReference;
@JsonCreator
public AzureCognitiveSearchMemoryRecord(
@JsonProperty(ID) String id,
@JsonProperty(TEXT) String text,
@JsonProperty(DESCRIPTION) String description,
@JsonProperty(ADDITIONAL_METADATA) String additionalMetadata,
@JsonProperty(EMBEDDING) List<Float> embedding,
@JsonProperty(EXTERNAL_SOURCE_NAME) String externalSourceName,
@JsonProperty(IS_REFERENCE) boolean isReference) {
this.id = id;
this.text = text;
this.description = description;
this.additionalMetadata = additionalMetadata;
this.embedding = embedding != null ? embedding : Collections.emptyList();
this.externalSourceName = externalSourceName;
this.isReference = isReference;
}
/**
* Record ID. The record is not filterable to save quota, also SK uses only semantic search.
*
* @return Record ID.
*/
public String getId() {
return id;
}
/**
* Content is stored here.
*
* @return Content is stored here, {@code null} if not set.
*/
public String getText() {
return text;
}
/**
* Optional description of the content, e.g. a title. This can be useful when indexing external
* data without pulling in the entire content.
*
* @return Optional description of the content, {@code null} if not set.
*/
public String getDescription() {
return description;
}
/**
* Additional metadata. Currently, this is a String where you could store serialized data as
* JSON. In future the design might change to allow storing named values and leverage filters.
*
* @return Additional metadata, {@code null} if not set.
*/
public String getAdditionalMetadata() {
return additionalMetadata;
}
/**
* Embedding vector.
*
* @return Embedding vector.
*/
public List<Float> getEmbedding() {
return embedding;
}
/**
* Name of the external source, in cases where the content and the Id are referenced to external
* information.
*
* @return Name of the external source, in cases where the content and the Id are referenced to
* external information, {@code null} if not set.
*/
public String getExternalSourceName() {
return externalSourceName;
}
/**
* Whether the record references external information.
*
* @return {@code true} if the record references external information, {@code false} otherwise.
*/
public boolean isReference() {
return isReference;
}
public MemoryRecordMetadata toMemoryRecordMetadata() {
return new MemoryRecordMetadata(
this.isReference,
decodeId(this.id),
this.text,
this.description,
this.externalSourceName,
this.additionalMetadata);
}
public static AzureCognitiveSearchMemoryRecord fromMemoryRecord(MemoryRecord record) {
return new AzureCognitiveSearchMemoryRecord(
record.getMetadata().getId(),
record.getMetadata().getText(),
record.getMetadata().getDescription(),
record.getMetadata().getAdditionalMetadata(),
record.getEmbedding().getVector(),
record.getMetadata().getExternalSourceName(),
record.getMetadata().isReference());
}
public MemoryRecord toMemoryRecord() {
return toMemoryRecord(true);
}
public MemoryRecord toMemoryRecord(boolean withEmbeddings) {
return new MemoryRecord(
this.toMemoryRecordMetadata(),
new Embedding(withEmbeddings ? this.embedding : Collections.emptyList()),
this.id,
null);
}
// ACS keys can contain only letters, digits, underscore, dash, equal sign, recommending
// to encode values with a URL-safe algorithm.
// <param name="realId">Original Id</param>
// <returns>Encoded id</returns>
static String encodeId(@Nullable String realId) {
if (realId == null) {
return "";
}
byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8));
return new String(bytes, StandardCharsets.UTF_8);
}
static String decodeId(@Nullable String encodedId) {
if (encodedId == null) {
return "";
}
byte[] bytes = Base64.getUrlDecoder().decode(encodedId.getBytes(StandardCharsets.UTF_8));
return new String(bytes, StandardCharsets.UTF_8);
}
}
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryStore.java
+465
View File
@@ -0,0 +1,465 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.connectors.memory.azurecognitivesearch;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.credential.TokenCredential;
import com.azure.core.http.rest.Response;
import com.azure.core.util.ClientOptions;
import com.azure.core.util.MetricsOptions;
import com.azure.core.util.TracingOptions;
import com.azure.search.documents.SearchAsyncClient;
import com.azure.search.documents.SearchDocument;
import com.azure.search.documents.indexes.SearchIndexAsyncClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
import com.azure.search.documents.indexes.models.HnswParameters;
import com.azure.search.documents.indexes.models.HnswVectorSearchAlgorithmConfiguration;
import com.azure.search.documents.indexes.models.LexicalAnalyzerName;
import com.azure.search.documents.indexes.models.SearchField;
import com.azure.search.documents.indexes.models.SearchFieldDataType;
import com.azure.search.documents.indexes.models.SearchIndex;
import com.azure.search.documents.indexes.models.VectorSearch;
import com.azure.search.documents.indexes.models.VectorSearchAlgorithmMetric;
import com.azure.search.documents.models.IndexDocumentsOptions;
import com.azure.search.documents.models.IndexingResult;
import com.azure.search.documents.models.SearchOptions;
import com.azure.search.documents.models.SearchQueryVector;
import com.microsoft.semantickernel.ai.embeddings.Embedding;
import com.microsoft.semantickernel.connectors.memory.azurecognitivesearch.AzureCognitiveSearchMemoryException.ErrorCodes;
import com.microsoft.semantickernel.memory.MemoryRecord;
import com.microsoft.semantickernel.memory.MemoryStore;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;
/**
* Semantic Memory implementation using Azure Cognitive Search. For more information about Azure
* Cognitive Search {@see https://learn.microsoft.com/azure/search/search-what-is-azure-search}
*/
public class AzureCognitiveSearchMemoryStore implements MemoryStore {
private static final String USER_AGENT = "Semantic-Kernel";
private static final String SEARCH_CONFIG_NAME = "searchConfig";
private final SearchIndexAsyncClient _adminClient;
private final Map<String, SearchAsyncClient> _clientsByIndex = new ConcurrentHashMap<>();
/// <summary>
/// Create a new instance of memory storage using Azure Cognitive Search.
/// </summary>
/// <param name="endpoint">Azure Cognitive Search URI, e.g.
// "https://contoso.search.windows.net"</param>
/// <param name="apiKey">API Key</param>
/**
* Create a new instance of memory storage using Azure Cognitive Search.
*
* @param endpoint Azure Cognitive Search URI, e.g. "https://contoso.search.windows.net"
* @param apiKey Azure API Key
*/
public AzureCognitiveSearchMemoryStore(@Nonnull String endpoint, @Nonnull String apiKey) {
this(
new SearchIndexClientBuilder()
.endpoint(endpoint)
.credential(new AzureKeyCredential(apiKey))
.clientOptions(clientOptions())
.buildAsyncClient());
}
/**
* Create a new instance of memory storage using Azure Cognitive Search.
*
* @param endpoint Azure Cognitive Search URI, e.g. "https://contoso.search.windows.net"
* @param credentials Azure service credentials
*/
public AzureCognitiveSearchMemoryStore(
@Nonnull String endpoint, @Nonnull TokenCredential credentials) {
this(
new SearchIndexClientBuilder()
.endpoint(endpoint)
.credential(credentials)
.clientOptions(clientOptions())
.buildAsyncClient());
}
/**
* Create a new instance of memory storage using Azure Cognitive Search.
*
* @param searchIndexAsyncClient the Azure search documents index client to use
*/
public AzureCognitiveSearchMemoryStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient) {
this._adminClient = searchIndexAsyncClient;
}
@Override
public Mono<Void> createCollectionAsync(@Nonnull String collectionName) {
// Indexes are created when sending a record. The creation requires the size of the
// embeddings.
return Mono.empty();
}
@Override
public Mono<List<String>> getCollectionsAsync() {
return this.getIndexesAsync();
}
@Override
public Mono<Boolean> doesCollectionExistAsync(@Nonnull String collectionName) {
String normalizedIndexName = normalizeIndexName(collectionName);
return getIndexesAsync()
.map(
list ->
list.stream()
.anyMatch(
name ->
name.equalsIgnoreCase(collectionName)
|| name.equalsIgnoreCase(
normalizedIndexName)));
}
@Override
public Mono<Void> deleteCollectionAsync(@Nonnull String collectionName) {
collectionName = normalizeIndexName(collectionName);
return _adminClient.deleteIndex(collectionName);
}
@Override
public Mono<String> upsertAsync(@Nonnull String collectionName, @Nonnull MemoryRecord record) {
collectionName = normalizeIndexName(collectionName);
return this.upsertRecordAsync(
collectionName, AzureCognitiveSearchMemoryRecord.fromMemoryRecord(record));
}
@Override
public Mono<Collection<String>> upsertBatchAsync(
@Nonnull String collectionName, Collection<MemoryRecord> records) {
if (records == null || records.isEmpty()) {
return Mono.just(Collections.emptyList());
}
collectionName = normalizeIndexName(collectionName);
List<AzureCognitiveSearchMemoryRecord> searchRecords =
records.stream()
.map(AzureCognitiveSearchMemoryRecord::fromMemoryRecord)
.collect(Collectors.toList());
return upsertBatchAsync(collectionName, searchRecords);
}
@Override
public Mono<MemoryRecord> getAsync(
@Nonnull String collectionName, @Nonnull String key, boolean withEmbedding) {
collectionName = normalizeIndexName(collectionName);
key = AzureCognitiveSearchMemoryRecord.encodeId(key);
SearchAsyncClient client = this.getSearchClient(collectionName);
return client.getDocumentWithResponse(key, AzureCognitiveSearchMemoryRecord.class, null)
.map(
response -> {
if (response.getStatusCode() == 404) {
throw new AzureCognitiveSearchMemoryException(
ErrorCodes.READ_FAILURE, "Index not found");
}
return response.getValue().toMemoryRecord(withEmbedding);
});
}
@Override
public Mono<Collection<MemoryRecord>> getBatchAsync(
@Nonnull String collectionName,
@Nonnull Collection<String> keys,
boolean withEmbedding) {
// TODO: there _must_ be a better way than one request per key!
return Flux.fromIterable(keys)
.flatMap(key -> getAsync(collectionName, key, withEmbedding))
.collect(Collectors.toList());
}
@Override
public Mono<Tuple2<MemoryRecord, Float>> getNearestMatchAsync(
@Nonnull String collectionName,
@Nonnull Embedding embedding,
float minRelevanceScore,
boolean withEmbedding) {
return getNearestMatchesAsync(
collectionName, embedding, 1, minRelevanceScore, withEmbedding)
.map(matches -> matches.iterator().next());
}
@Override
public Mono<Collection<Tuple2<MemoryRecord, Float>>> getNearestMatchesAsync(
@Nonnull String collectionName,
@Nonnull Embedding embedding,
int limit,
float minRelevanceScore,
boolean withEmbedding) {
collectionName = normalizeIndexName(collectionName);
SearchAsyncClient client = getSearchClient(collectionName);
SearchQueryVector searchVector =
new SearchQueryVector()
.setKNearestNeighborsCount(limit)
.setFields(AzureCognitiveSearchMemoryRecord.EMBEDDING)
.setValue(embedding.getVector());
SearchOptions searchOptions = new SearchOptions().setVectors(searchVector);
return client.search(null, searchOptions)
.filter(result -> (double) minRelevanceScore >= result.getScore())
.map(
result -> {
MemoryRecord memoryRecord =
result.getDocument(AzureCognitiveSearchMemoryRecord.class)
.toMemoryRecord(withEmbedding);
float score = (float) result.getScore();
return Tuples.of(memoryRecord, score);
})
.collect(Collectors.toList());
}
@Override
public Mono<Void> removeAsync(@Nonnull String collectionName, @Nonnull String key) {
return removeBatchAsync(collectionName, Collections.singleton(key));
}
@Override
public Mono<Void> removeBatchAsync(
@Nonnull String collectionName, @Nonnull Collection<String> keys) {
collectionName = normalizeIndexName(collectionName);
SearchAsyncClient client = this.getSearchClient(collectionName);
List<SearchDocument> documents =
keys.stream()
.map(AzureCognitiveSearchMemoryRecord::encodeId)
.map(
key -> {
SearchDocument searchDocument = new SearchDocument();
searchDocument.put("Id", key); // TODO: Id should be a constant
return searchDocument;
})
.collect(Collectors.toList());
return client.deleteDocuments(documents).then();
}
static List<SearchField> searchFields(String configName, int embeddingSize) {
return Arrays.asList(
new SearchField(AzureCognitiveSearchMemoryRecord.ID, SearchFieldDataType.STRING)
.setKey(true)
.setFilterable(false),
new SearchField(
AzureCognitiveSearchMemoryRecord.EMBEDDING,
SearchFieldDataType.collection(SearchFieldDataType.SINGLE))
.setSearchable(true)
.setVectorSearchDimensions(embeddingSize)
.setVectorSearchConfiguration(configName),
new SearchField(AzureCognitiveSearchMemoryRecord.TEXT, SearchFieldDataType.STRING)
.setSearchable(true)
.setFacetable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField(
AzureCognitiveSearchMemoryRecord.DESCRIPTION,
SearchFieldDataType.STRING)
.setSearchable(true)
.setFacetable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField(
AzureCognitiveSearchMemoryRecord.ADDITIONAL_METADATA,
SearchFieldDataType.STRING)
.setSearchable(true)
.setFacetable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField(
AzureCognitiveSearchMemoryRecord.EXTERNAL_SOURCE_NAME,
SearchFieldDataType.STRING)
.setSearchable(true)
.setFacetable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField(
AzureCognitiveSearchMemoryRecord.IS_REFERENCE,
SearchFieldDataType.BOOLEAN)
.setFilterable(true)
.setFacetable(true));
}
/**
* Create a new search index.
*
* @param indexName Index name
* @param embeddingSize Size of the embedding vector
* @return A Mono that completes when the index is created
*/
private Mono<Response<SearchIndex>> createIndexAsync(
@Nonnull String indexName, int embeddingSize) {
if (embeddingSize < 1) {
throw new AzureCognitiveSearchMemoryException(
ErrorCodes.INVALID_EMBEDDING_SIZE, "the value must be greater than zero");
}
String normalizedIndexName = normalizeIndexName(indexName);
VectorSearch vectorSearch =
new VectorSearch()
.setAlgorithmConfigurations(
Collections.singletonList(
new HnswVectorSearchAlgorithmConfiguration(
SEARCH_CONFIG_NAME)
.setParameters(
new HnswParameters()
.setMetric(
VectorSearchAlgorithmMetric
.COSINE))));
SearchIndex newIndex =
new SearchIndex(normalizedIndexName)
.setFields(searchFields(SEARCH_CONFIG_NAME, embeddingSize))
.setVectorSearch(vectorSearch);
return _adminClient.createIndexWithResponse(newIndex);
}
private Mono<List<String>> getIndexesAsync() {
return _adminClient.listIndexes().map(SearchIndex::getName).collect(Collectors.toList());
}
private Mono<String> upsertRecordAsync(
@Nonnull String indexName, @Nonnull AzureCognitiveSearchMemoryRecord record) {
return upsertBatchAsync(indexName, Collections.singletonList(record))
.map(Collection::iterator)
.map(Iterator::next);
}
private Mono<Collection<String>> upsertBatchAsync(
@Nonnull String indexName, @Nonnull List<AzureCognitiveSearchMemoryRecord> records) {
if (records.isEmpty()) {
return Mono.just(Collections.emptyList());
}
List<SearchDocument> documents =
records.stream()
.map(
record -> {
SearchDocument searchDocument = new SearchDocument();
searchDocument.put(
AzureCognitiveSearchMemoryRecord.ID,
AzureCognitiveSearchMemoryRecord.encodeId(
record.getId()));
searchDocument.put(
AzureCognitiveSearchMemoryRecord.TEXT,
record.getText());
searchDocument.put(
AzureCognitiveSearchMemoryRecord.DESCRIPTION,
record.getDescription());
searchDocument.put(
AzureCognitiveSearchMemoryRecord.EMBEDDING,
record.getEmbedding());
searchDocument.put(
AzureCognitiveSearchMemoryRecord.ADDITIONAL_METADATA,
record.getAdditionalMetadata());
searchDocument.put(
AzureCognitiveSearchMemoryRecord.EXTERNAL_SOURCE_NAME,
record.getExternalSourceName());
searchDocument.put(
AzureCognitiveSearchMemoryRecord.IS_REFERENCE,
record.isReference());
return searchDocument;
})
.collect(Collectors.toList());
IndexDocumentsOptions options = new IndexDocumentsOptions().setThrowOnAnyError(true);
int embeddingSize =
records.stream()
.map(AzureCognitiveSearchMemoryRecord::getEmbedding)
.map(List::size)
.max(Integer::compareTo)
.orElse(0);
String normalizedIndexName = normalizeIndexName(indexName);
SearchAsyncClient client = this.getSearchClient(normalizedIndexName);
return client.uploadDocumentsWithResponse(documents, options)
.map(
response -> {
if (response.getStatusCode() == 404) {
throw new AzureCognitiveSearchMemoryException(
ErrorCodes.INVALID_INDEX_NAME);
}
return response;
})
.onErrorResume(
e ->
createIndexAsync(normalizedIndexName, embeddingSize)
.then(
client.uploadDocumentsWithResponse(
documents, options)))
.map(
response ->
response.getValue().getResults().stream()
.map(IndexingResult::getKey)
.collect(Collectors.toList()));
}
/**
* Get a search client for the index specified. Note: the index might not exist, but we avoid
* checking everytime and the extra latency.
*
* @param indexName Index name
* @return Search client ready to read/write
*/
private SearchAsyncClient getSearchClient(@Nonnull String indexName) {
String normalizedIndexName = normalizeIndexName(indexName);
return _clientsByIndex.computeIfAbsent(
normalizedIndexName, _adminClient::getSearchAsyncClient);
}
/// <summary>
/// Options used by the Azure Cognitive Search client, e.g. User Agent.
/// See also
// https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/src/DiagnosticsOptions.cs
/// </summary>
private static ClientOptions clientOptions() {
return new ClientOptions()
.setTracingOptions(new TracingOptions())
.setMetricsOptions(new MetricsOptions())
.setApplicationId(USER_AGENT);
}
// Index names cannot contain special chars. We use this rule to replace a few common ones
// with an underscore and reduce the chance of errors. If other special chars are used, we leave
// it
// to the service to throw an error.
// Note:
// - replacing chars introduces a small chance of conflicts, e.g. "the-user" and "the_user".
// - we should consider whether making this optional and leave it to the developer to handle.
//
// Java 8 does not support \s for whitespace, so we have to list them all.
private static final String whitespace = " |\\t|\\n|\\x0B|\\f|\\r";
private static final String s_replaceIndexNameSymbolsRegex = "[" + whitespace + "|\\|/|.|_|:]";
// Normalize index name to match ACS rules.
// See https://learn.microsoft.com/en-us/rest/api/searchservice/Create-Index
// The method doesn't handle all the error scenarios, leaving it to the service
// to throw an error for edge cases not handled locally.
// <param name="indexName">Value to normalize</param>
// <returns>Normalized name</returns>
private static String normalizeIndexName(String indexName) {
if (indexName.length() > 128) {
throw new IllegalArgumentException("The collection name cannot exceed 128 chars");
}
indexName = indexName.toLowerCase(Locale.ROOT);
return indexName.replaceAll(s_replaceIndexNameSymbolsRegex, "-");
}
}
java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/main/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchRecord.java
-131
View File
@@ -1,131 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
package com.microsoft.semantickernel.connectors.memory.azurecognitivesearch;
import com.azure.search.documents.indexes.SearchableField;
import com.azure.search.documents.indexes.SimpleField;
import com.azure.search.documents.indexes.models.LexicalAnalyzerName;
import com.azure.search.documents.indexes.models.SearchField;
import com.azure.search.documents.indexes.models.SearchFieldDataType;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Arrays;
import java.util.List;
/**
* Azure Cognitive Search record and index definition. Note: once defined, index cannot be modified.
*/
public final class AzureCognitiveSearchRecord {
@SimpleField(isKey = true, isFilterable = false)
private final String id;
@SearchableField(analyzerName = "standard.lucene")
private final String text;
@SearchableField(analyzerName = "standard.lucene")
private final String description;
@SearchableField(analyzerName = "standard.lucene")
private final String additionalMetadata;
@SimpleField(isFilterable = false)
private final String externalSourceName;
@SimpleField(isFilterable = false)
private final boolean isReference;
@JsonCreator
public AzureCognitiveSearchRecord(
@JsonProperty("Id") String id,
@JsonProperty("Text") String text,
@JsonProperty("Description") String description,
@JsonProperty("AdditionalMetadata") String additionalMetadata,
@JsonProperty("ExternalSourceName") String externalSourceName,
@JsonProperty("Reference") boolean isReference) {
this.id = id;
this.text = text;
this.description = description;
this.additionalMetadata = additionalMetadata;
this.externalSourceName = externalSourceName;
this.isReference = isReference;
}
/**
* Record ID. The record is not filterable to save quota, also SK uses only semantic search.
*
* @return Record ID.
*/
public String getId() {
return id;
}
/**
* Content is stored here.
*
* @return Content is stored here, {@code null} if not set.
*/
public String getText() {
return text;
}
/**
* Optional description of the content, e.g. a title. This can be useful when indexing external
* data without pulling in the entire content.
*
* @return Optional description of the content, {@code null} if not set.
*/
public String getDescription() {
return description;
}
/**
* Additional metadata. Currently, this is a String where you could store serialized data as
* JSON. In future the design might change to allow storing named values and leverage filters.
*
* @return Additional metadata, {@code null} if not set.
*/
public String getAdditionalMetadata() {
return additionalMetadata;
}
/**
* Name of the external source, in cases where the content and the Id are referenced to external
* information.
*
* @return Name of the external source, in cases where the content and the Id are referenced to
* external information, {@code null} if not set.
*/
public String getExternalSourceName() {
return externalSourceName;
}
/**
* Whether the record references external information.
*
* @return {@code true} if the record references external information, {@code false} otherwise.
*/
public boolean isReference() {
return isReference;
}
// TODO: Potentially remove after resolution of
// https://github.com/Azure/azure-sdk-for-java/issues/35584
static List<SearchField> searchFields() {
return Arrays.asList(
new SearchField("Id", SearchFieldDataType.STRING).setKey(true).setFilterable(false),
new SearchField("Text", SearchFieldDataType.STRING)
.setSearchable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField("Description", SearchFieldDataType.STRING)
.setSearchable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField("AdditionalMetadata", SearchFieldDataType.STRING)
.setSearchable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField("ExternalSourceName", SearchFieldDataType.STRING)
.setSearchable(true)
.setAnalyzerName(LexicalAnalyzerName.EN_LUCENE),
new SearchField("Reference", SearchFieldDataType.BOOLEAN).setFilterable(true));
}
// TODO: add one more field with the vector, float array, mark it as searchable
}
java/semantickernel-core/src/test/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryTests.java → java/connectors/semantickernel-connectors-memory-azurecognitivesearch/src/test/java/com/microsoft/semantickernel/connectors/memory/azurecognitivesearch/AzureCognitiveSearchMemoryTests.java
+30 -6
View File
@@ -17,19 +17,23 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient;
import com.azure.search.documents.indexes.SearchIndexClientBuilder;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.SKBuilders;
import com.microsoft.semantickernel.ai.embeddings.Embedding;
import com.microsoft.semantickernel.ai.embeddings.EmbeddingGeneration;
import com.microsoft.semantickernel.memory.MemoryQueryResult;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.function.Function;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import reactor.core.publisher.Mono;
// TODO: This class is in the core package to avoid a
// circular dependency with the connectors module.
@Disabled("Pending https://github.com/microsoft/semantic-kernel/issues/1797")
class AzureCognitiveSearchMemoryTests {
// com.azure.core.implementation.http.rest.RestProxyBase.RestProxyBase
@@ -56,7 +60,7 @@ class AzureCognitiveSearchMemoryTests {
}
}
// Kernel is configured with a real AzureCognitiveSearchMemory which is backed
// Kernel is configured with a real AzureCognitiveSearchMemoryStore which is backed
// by a mocked HttpPipeline. The HttpPipeline is mocked to return a mocked
// HttpResponse whose body and status code are configured by the test.
Kernel kernel;
@@ -82,21 +86,38 @@ class AzureCognitiveSearchMemoryTests {
@BeforeEach
void setUp() {
String apiKey = "fake-key";
HttpPipeline pipeline = setUpHttpPipeline();
SearchIndexAsyncClient searchIndexAsyncClient =
new SearchIndexClientBuilder()
.endpoint("https://fake-random-test-host/fake-path")
.credential(new AzureKeyCredential("fake-key"))
.credential(new AzureKeyCredential(apiKey))
.pipeline(pipeline)
.httpLogOptions(
new HttpLogOptions()
.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS))
.buildAsyncClient();
EmbeddingGeneration<String> embeddingGeneration =
new EmbeddingGeneration<String>() {
List<Embedding> embeddings = new ArrayList<>();
{
embeddings.add(new Embedding(Arrays.asList(1.0f, 2.0f, 3.0f)));
}
@Override
public Mono<List<Embedding>> generateEmbeddingsAsync(List<String> data) {
return Mono.just(embeddings);
}
};
kernel =
SKBuilders.kernel()
.withMemory(new AzureCognitiveSearchMemory(searchIndexAsyncClient))
.withMemoryStorage(
new AzureCognitiveSearchMemoryStore(searchIndexAsyncClient))
.withDefaultAIService(embeddingGeneration)
.build();
}
@@ -116,8 +137,11 @@ class AzureCognitiveSearchMemoryTests {
private Mono<HttpResponse> answerHttpRequest(InvocationOnMock invocation) {
HttpRequest request = invocation.getArgument(0);
String query = request.getUrl().getQuery();
int offset = query.indexOf("api-version");
assertTrue(offset >= 0);
// HttpResponse is based on this api-version
assertTrue(request.getUrl().getQuery().contains("api-version=2021-04-30-Preview"));
assertEquals("api-version=2023-07-01-Preview", query.substring(offset));
// System.err.println(request.getHttpMethod() + " " + request.getUrl() + "\n\t" +
// request.getHeaders());
java/samples/sample-code/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example14_SemanticMemory.java
+12 -6
View File
@@ -4,7 +4,7 @@ package com.microsoft.semantickernel.samples.syntaxexamples;
import com.microsoft.semantickernel.Kernel;
import com.microsoft.semantickernel.SKBuilders;
import com.microsoft.semantickernel.SamplesConfig;
import com.microsoft.semantickernel.connectors.memory.azurecognitivesearch.AzureCognitiveSearchMemory;
import com.microsoft.semantickernel.connectors.memory.azurecognitivesearch.AzureCognitiveSearchMemoryStore;
import com.microsoft.semantickernel.memory.VolatileMemoryStore;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -32,14 +32,21 @@ public class Example14_SemanticMemory {
System.out.println("======== Semantic Memory using Azure Cognitive Search ========");
System.out.println("==============================================================");
/* This example leverages Azure Cognitive Search to provide SK with Semantic Memory.
var openAIAsyncClient = SamplesConfig.getClient();
/* This example leverages Azure Cognitive Search to provide SK with Semantic Memory.
*
* Azure Cognitive Search automatically indexes your data semantically, so you don't
* need to worry about embedding generation.
*/
var kernelWithACS = SKBuilders.kernel()
.withMemory(new AzureCognitiveSearchMemory(System.getenv("ACS_ENDPOINT"), System.getenv("ACS_API_KEY")))
.build();
var kernelWithACS = SKBuilders.kernel()
.withMemoryStorage(
new AzureCognitiveSearchMemoryStore(System.getenv("ACS_ENDPOINT"), System.getenv("ACS_API_KEY")))
.withDefaultAIService(SKBuilders.textEmbeddingGeneration()
.withOpenAIClient(openAIAsyncClient)
.withModelId("text-embedding-ada-002")
.build())
.build();
runExampleAsync(kernelWithACS).block();
@@ -55,7 +62,6 @@ public class Example14_SemanticMemory {
* You can replace VolatileMemoryStore with Qdrant (see QdrantMemoryStore connector)
* or implement your connectors for Pinecone, Vespa, Postgres + pgvector, SQLite VSS, etc.
*/
var openAIAsyncClient = SamplesConfig.getClient();
var kernelWithCustomDb = SKBuilders.kernel()
.withDefaultAIService(SKBuilders.textEmbeddingGeneration()
java/semantickernel-bom/pom.xml
+1 -1
View File
@@ -142,7 +142,7 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-search-documents</artifactId>
<version>11.6.0-beta.6</version>
<version>11.6.0-beta.9</version>
</dependency>
<dependency>
<groupId>jakarta.inject</groupId>
java/semantickernel-core/pom.xml
-5
View File
@@ -86,11 +86,6 @@
<artifactId>semantickernel-planners</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.microsoft.semantic-kernel</groupId>
<artifactId>semantickernel-connectors-memory-azurecognitivesearch</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>jakarta.inject</groupId>
java/semantickernel-core/src/main/java/com/microsoft/semantickernel/memory/VolatileMemoryStore.java
+4 -3
View File
@@ -70,8 +70,8 @@ public class VolatileMemoryStore implements MemoryStore {
// Contract:
// Does not guarantee that the collection exists.
// getCollection throws MemoryException if the collection does not exist.
Map<String, MemoryRecord> collection = getCollection(collectionName);
Map<String, MemoryRecord> collection =
_store.computeIfAbsent(collectionName, k -> new ConcurrentHashMap<>());
String key = record.getMetadata().getId();
// Assumption is that MemoryRecord will always have a non-null id.
@@ -93,7 +93,8 @@ public class VolatileMemoryStore implements MemoryStore {
return Mono.fromCallable(
() -> {
Map<String, MemoryRecord> collection = getCollection(collectionName);
Map<String, MemoryRecord> collection =
_store.computeIfAbsent(collectionName, k -> new ConcurrentHashMap<>());
Set<String> keys = new HashSet<>();
records.forEach(
record -> {
java/semantickernel-core/src/test/java/com/microsoft/semantickernel/memory/VolatileMemoryStoreTests.java
+2
View File
@@ -16,6 +16,7 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import reactor.core.publisher.Flux;
import reactor.util.function.Tuple2;
@@ -102,6 +103,7 @@ class VolatileMemoryStoreTests {
}
@Test
@Disabled("inconsistency between memory store implementations")
void itCannotInsertIntoNonExistentCollectionAsync() {
// Arrange
java/spotbugs-exclude.xml
+10
View File
@@ -100,4 +100,14 @@
<Bug pattern="EI_EXPOSE_REP2"/>
</Match>
<Match>
<Class name="com.microsoft.semantickernel.connectors.memory.azurecognitivesearch.AzureCognitiveSearchMemoryRecord"/>
<Bug pattern="EI_EXPOSE_REP"/>
</Match>
<Match>
<Class name="com.microsoft.semantickernel.connectors.memory.azurecognitivesearch.AzureCognitiveSearchMemoryStore"/>
<Bug pattern="EI_EXPOSE_REP2"/>
</Match>
</FindBugsFilter>