From 2110556d5534e2ca33ec80ed0191323a37c9d078 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:44:04 +0000 Subject: [PATCH] Java: Update redis and spring-starter versions (#4998) ### Motivation and Context ### Description ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone :smile: --- .../pom.xml | 32 +++------ .../memory/redis/JsonMemoryRecord.java | 13 +--- .../memory/redis/RedisMemoryStore.java | 69 ++++++++++++------- .../semantickernel-spring-starter/pom.xml | 2 +- .../microsoft/semantickernel/SKException.java | 2 +- java/semantickernel-bom/pom.xml | 7 ++ 6 files changed, 65 insertions(+), 60 deletions(-) diff --git a/java/connectors/semantickernel-connectors-memory-redis/pom.xml b/java/connectors/semantickernel-connectors-memory-redis/pom.xml index eb419a16ce..51a340d01f 100644 --- a/java/connectors/semantickernel-connectors-memory-redis/pom.xml +++ b/java/connectors/semantickernel-connectors-memory-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 0.2.10-alpha-SNAPSHOT + 0.2.14-alpha-SNAPSHOT ../../pom.xml @@ -13,15 +13,6 @@ Semantic Kernel Redis Memory Connector Defines connector for Redis to be used with the Semantic Kernel - - UTF-8 - UTF-8 - 17 - 17 - 17 - 17 - - com.fasterxml.jackson.core @@ -40,15 +31,18 @@ org.mockito mockito-core + test org.junit.jupiter junit-jupiter + test redis.clients jedis 5.1.0 + provided com.microsoft.semantic-kernel @@ -56,19 +50,9 @@ ${project.version} compile + + com.github.spotbugs + spotbugs-annotations + - - - - org.apache.maven.plugins - maven-compiler-plugin - - - 17 - 17 - 17 - - - - \ No newline at end of file diff --git a/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java b/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java index e850329c2e..68ca9d4003 100644 --- a/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java +++ b/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java @@ -8,7 +8,6 @@ import com.microsoft.semantickernel.memory.MemoryRecord; import com.microsoft.semantickernel.memory.MemoryRecordMetadata; import java.nio.charset.StandardCharsets; import java.util.*; -import javax.annotation.Nullable; /** Represents an entry in the Semantic Kernel Memory Table. */ public class JsonMemoryRecord { @@ -92,7 +91,7 @@ public class JsonMemoryRecord { * @return Embedding vector. */ public List getEmbedding() { - return embedding; + return Collections.unmodifiableList(embedding); } /** @@ -152,18 +151,12 @@ public class JsonMemoryRecord { // to encode values with a URL-safe algorithm. // Original Id // Encoded id - static String encodeId(@Nullable String realId) { - if (realId == null) { - return ""; - } + static String encodeId(String realId) { byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); return new String(bytes, StandardCharsets.UTF_8); } - static String decodeId(@Nullable String encodedId) { - if (encodedId == null) { - return ""; - } + static String decodeId(String encodedId) { byte[] bytes = Base64.getUrlDecoder().decode(encodedId.getBytes(StandardCharsets.UTF_8)); return new String(bytes, StandardCharsets.UTF_8); } diff --git a/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java b/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java index ac56b18cfe..6a40d3d57a 100644 --- a/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java +++ b/java/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java @@ -1,27 +1,42 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.memory.redis; +import com.microsoft.semantickernel.SKException; import com.microsoft.semantickernel.ai.embeddings.Embedding; import com.microsoft.semantickernel.memory.MemoryException; import com.microsoft.semantickernel.memory.MemoryException.ErrorCodes; import com.microsoft.semantickernel.memory.MemoryRecord; import com.microsoft.semantickernel.memory.MemoryStore; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.text.MessageFormat; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; import redis.clients.jedis.JedisPooled; import redis.clients.jedis.exceptions.JedisDataException; import redis.clients.jedis.resps.ScanResult; -import redis.clients.jedis.search.*; +import redis.clients.jedis.search.Document; +import redis.clients.jedis.search.IndexDefinition; +import redis.clients.jedis.search.IndexOptions; +import redis.clients.jedis.search.Query; +import redis.clients.jedis.search.Schema; +import redis.clients.jedis.search.SearchResult; import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm; /** @@ -43,11 +58,12 @@ public class RedisMemoryStore implements MemoryStore { private static final String OK = "OK"; private final JedisPooled client; - private Integer vectorSize = DefaultVectorSize; - private VectorAlgorithm vectorIndexAlgorithm = DefaultIndexAlgorithm; - private String vectorDistanceMetric = DefaultDistanceMetric; - private String vectorType = DefaultVectorType; - private Integer queryDialect; + private final String vectorDistanceMetric; + + @SuppressFBWarnings("SS_SHOULD_BE_STATIC") + private final String vectorType = DefaultVectorType; + + private final Integer queryDialect; public static void isXinRange(Integer x, Integer lower, Integer upper, String message) { if (lower > x || x > upper) { @@ -61,15 +77,14 @@ public class RedisMemoryStore implements MemoryStore { * @param client A Redis Database client connection. * @param vectorSize Embedding vector size, defaults to 1536 - framework model dependent - * ada-002. - * @param vectorIndexAlgorithm Indexing algorithm for vectors, defaults to "HNSW" * @param vectorDistanceMetric Metric for measuring vector distances, defaults to "COSINE" * @param queryDialect Query dialect, must be 2 or greater for vector similarity searching, * defaults to 2 */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public RedisMemoryStore( JedisPooled client, Integer vectorSize, - VectorAlgorithm vectorIndexAlgorithm, String vectorDistanceMetric, Integer queryDialect) { @@ -85,9 +100,6 @@ public class RedisMemoryStore implements MemoryStore { "Invalid query dialect: {x}. Query dialect must be in the range 2-4."); this.client = client; - - this.vectorSize = vectorSize; - this.vectorIndexAlgorithm = vectorIndexAlgorithm; this.vectorDistanceMetric = vectorDistanceMetric; this.queryDialect = queryDialect; } @@ -121,9 +133,6 @@ public class RedisMemoryStore implements MemoryStore { "Invalid query dialect: {x}. Query dialect must be in the range 2-4."); this.client = new JedisPooled(connectionString); - - this.vectorSize = vectorSize; - this.vectorIndexAlgorithm = vectorIndexAlgorithm; this.vectorDistanceMetric = vectorDistanceMetric; this.queryDialect = queryDialect; } @@ -136,6 +145,13 @@ public class RedisMemoryStore implements MemoryStore { * @return A Redis key that identifies a particular Hash of Json object. */ private static String getRedisKey(String collectionName, String key) { + if (key == null) { + throw new SKException("cannot create a key with a null value"); + } + + if (collectionName == null) { + throw new SKException("cannot form a redis key with a null collection name"); + } String name = collectionName.toLowerCase(Locale.ROOT); String id = JsonMemoryRecord.encodeId(key); @@ -171,10 +187,16 @@ public class RedisMemoryStore implements MemoryStore { @Override public Mono> getCollectionsAsync() { - return Mono.just( - getIndexesAsync().block().stream() - .map(name -> name.substring(0, name.length() - "-sk-idx".length())) - .collect(Collectors.toList())); + return getIndexesAsync() + .map( + names -> + names.stream() + .map( + name -> + name.substring( + 0, + name.length() - "-sk-idx".length())) + .collect(Collectors.toList())); } /** @@ -304,12 +326,11 @@ public class RedisMemoryStore implements MemoryStore { Objects.requireNonNull(collectionName); // ACS issues one query per key; redis has several possible calls, fastest will be hkeys // followed by hvals - Collection records = - keys.stream() - .map(key -> getAsync(collectionName, key, withEmbeddings).block()) - .collect(Collectors.toList()); - return Mono.just(records); + return Flux.fromIterable(keys) + .flatMap(key -> getAsync(collectionName, key, withEmbeddings)) + .collectList() + .map(records -> records); } @Override diff --git a/java/samples/semantickernel-spring-starter/pom.xml b/java/samples/semantickernel-spring-starter/pom.xml index e071f3837f..e678cc9d8a 100644 --- a/java/samples/semantickernel-spring-starter/pom.xml +++ b/java/samples/semantickernel-spring-starter/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel samples - 0.2.10-alpha-SNAPSHOT + 0.2.14-alpha-SNAPSHOT ../pom.xml diff --git a/java/semantickernel-api/src/main/java/com/microsoft/semantickernel/SKException.java b/java/semantickernel-api/src/main/java/com/microsoft/semantickernel/SKException.java index 28c89779ba..863b8cd5bb 100644 --- a/java/semantickernel-api/src/main/java/com/microsoft/semantickernel/SKException.java +++ b/java/semantickernel-api/src/main/java/com/microsoft/semantickernel/SKException.java @@ -18,7 +18,7 @@ public class SKException extends RuntimeException { * * @param message A string that describes the error. */ - protected SKException(@Nullable String message) { + public SKException(@Nullable String message) { super(message); } diff --git a/java/semantickernel-bom/pom.xml b/java/semantickernel-bom/pom.xml index 14654df49a..e9e0d2f5b1 100644 --- a/java/semantickernel-bom/pom.xml +++ b/java/semantickernel-bom/pom.xml @@ -171,6 +171,13 @@ postgresql 42.6.0 + + + + com.github.spotbugs + spotbugs-annotations + 4.8.3 +