Ssdeep overlap scorer update (#3009)

* Add cache for ngrams of query hashes. Prepare SSDeepNgramOverlapScorer to use cache.

* Revert "Add cache for ngrams of query hashes. Prepare SSDeepNgramOverlapScorer to use cache."

This reverts commit 7c9f386aec.

* Revert cache changes as they are not useful for overlap scoring. Overlap scoring changes in progress.

* Calculate overlapping ngrams via substring comparison

* Apply formatting to ssdeep overlap scoring changes

* Remove unused import in SSDeepNGramOverlapScorer

* Change location of ngram overlap calculation method and add recommended efficieny changes.

warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepScoringFunction.java

@@ -42,7 +42,7 @@ public class SSDeepScoringFunction implements Function<Map.Entry<NGramTuple,SSDe
         this.minScoreThreshold = readOptionalMinScoreThreshold(config.getQuery());
 
         this.editDistanceScorer = new SSDeepHashEditDistanceScorer(maxRepeatedCharacters);
-        this.ngramOverlapScorer = new SSDeepNGramOverlapScorer(config.getNGramSize(), maxRepeatedCharacters, config.getMinHashSize());
+        this.ngramOverlapScorer = new SSDeepNGramOverlapScorer(config.getNGramSize());
     }
 
     /**

warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java

@@ -70,8 +70,7 @@ public class SSDeepSimilarityQueryTransformerTest {
     @Test
     public void transformTest() {
 
-        final SSDeepHashScorer<Set<NGramTuple>> ngramOverlapScorer = new SSDeepNGramOverlapScorer(NGramGenerator.DEFAULT_NGRAM_SIZE,
-                        SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS, NGramGenerator.DEFAULT_MIN_HASH_SIZE);
+        final SSDeepHashScorer<Set<NGramTuple>> ngramOverlapScorer = new SSDeepNGramOverlapScorer(NGramGenerator.DEFAULT_NGRAM_SIZE);
         final SSDeepHashScorer<Integer> editDistanceScorer = new SSDeepHashEditDistanceScorer(SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS);
 
         final SSDeepHash query = SSDeepHash.parse(TEST_SSDEEPS[2]);

warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepNGramOverlapScorer.java

@@ -1,5 +1,6 @@
 package datawave.util.ssdeep;
 
+import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -8,17 +9,68 @@ import java.util.Set;
  */
 public class SSDeepNGramOverlapScorer implements SSDeepHashScorer<Set<NGramTuple>> {
 
-    NGramGenerator generator;
+    private final int ngramSize;
 
-    public SSDeepNGramOverlapScorer(int ngramSize, int maxRepeatedChars, int minHashSize) {
-        generator = new NGramGenerator(ngramSize, maxRepeatedChars, minHashSize);
+    public SSDeepNGramOverlapScorer(int ngramSize) {
+        this.ngramSize = ngramSize;
     }
 
+    /**
+     * Calculate the number of ngrams shared between a part of ssdeep hashes. Considers chunk sizes along with the single chunk / double chunk of each has as
+     * appropriate.
+     *
+     * @param signature1
+     *            the first ssdeep hash for comparison
+     * @param signature2
+     *            the second ssdeep hash for comparison
+     * @return the set of ngrams shared between each ssdeep hash, labeled with chunk size.
+     */
     public Set<NGramTuple> apply(SSDeepHash signature1, SSDeepHash signature2) {
-        Set<NGramTuple> ngrams1 = generator.generateNgrams(signature1);
-        Set<NGramTuple> ngrams2 = generator.generateNgrams(signature2);
+        if (signature1.getChunkSize() == signature2.getChunkSize()) {
+            Set<NGramTuple> ngrams = new HashSet<>();
 
-        ngrams1.retainAll(ngrams2);
-        return ngrams1;
+            ngrams.addAll(calculateOverlappingNGrams(ngramSize, signature1.getChunkSize(), signature1.getChunk(), signature2.getChunk()));
+
+            if (signature1.hasDoubleChunk() && signature2.hasDoubleChunk()) {
+                ngrams.addAll(calculateOverlappingNGrams(ngramSize, signature1.getDoubleChunkSize(), signature1.getDoubleChunk(), signature2.getDoubleChunk()));
+            }
+
+            return ngrams;
+        } else if (signature1.hasDoubleChunk() && (signature1.getDoubleChunkSize() == signature2.getChunkSize())) {
+            return calculateOverlappingNGrams(ngramSize, signature1.getDoubleChunkSize(), signature1.getDoubleChunk(), signature2.getChunk());
+        } else if (signature2.hasDoubleChunk() && (signature1.getChunkSize() == signature2.getDoubleChunkSize())) {
+            return calculateOverlappingNGrams(ngramSize, signature1.getChunkSize(), signature1.getChunk(), signature2.getDoubleChunk());
+        } else {
+            return new HashSet<NGramTuple>();
+        }
+    }
+
+    /**
+     * Calculate the number of ngrams shared amongst both chunks. Each chunk must have the same ngram size or the comparison will be invalid. For efficiency,
+     * the shorter chunk will drive the comparison.
+     *
+     * @param ngramSize
+     *            the size of ngrams to compare
+     * @param chunkSize
+     *            the chunk size of the ngrams we are comparing, both chunks must have the same size.
+     * @param chunk1
+     *            the first chunk to compare
+     * @param chunk2
+     *            the second chunk to compare
+     * @return a set of ngrams shared between each chunk, labeled with chunk size.
+     */
+    private static Set<NGramTuple> calculateOverlappingNGrams(int ngramSize, int chunkSize, String chunk1, String chunk2) {
+        Set<NGramTuple> overlappingNGrams = new HashSet<>();
+        final String smallChunk = chunk1.length() <= chunk2.length() ? chunk1 : chunk2;
+        final String largeChunk = chunk2.equals(smallChunk) ? chunk1 : chunk2;
+        final int ngramCount = smallChunk.length() - ngramSize;
+
+        for (int i = 0; i <= ngramCount; i++) {
+            String ngram = smallChunk.substring(i, i + ngramSize);
+            if (largeChunk.contains(ngram)) {
+                overlappingNGrams.add(new NGramTuple(chunkSize, ngram));
+            }
+        }
+        return overlappingNGrams;
     }
 }

warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java

@@ -30,7 +30,7 @@ public class SSDeepNGramOverlapScorerTest {
 
     @Test
     public void testCompare() {
-        SSDeepHashScorer<Set<NGramTuple>> scorer = new SSDeepNGramOverlapScorer(7, 3, 6);
+        SSDeepHashScorer<Set<NGramTuple>> scorer = new SSDeepNGramOverlapScorer(7);
         for (int i = 0; i < testData.length; i++) {
             SSDeepHash queryHash = SSDeepHash.parse(testData[i][0]);
             SSDeepHash targetHash = SSDeepHash.parse(testData[i][1]);