diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepScoringFunction.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepScoringFunction.java index f3b446690d..980fefdbb8 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepScoringFunction.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepScoringFunction.java @@ -42,7 +42,7 @@ public class SSDeepScoringFunction implements Function> ngramOverlapScorer = new SSDeepNGramOverlapScorer(NGramGenerator.DEFAULT_NGRAM_SIZE, - SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS, NGramGenerator.DEFAULT_MIN_HASH_SIZE); + final SSDeepHashScorer> ngramOverlapScorer = new SSDeepNGramOverlapScorer(NGramGenerator.DEFAULT_NGRAM_SIZE); final SSDeepHashScorer editDistanceScorer = new SSDeepHashEditDistanceScorer(SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS); final SSDeepHash query = SSDeepHash.parse(TEST_SSDEEPS[2]); diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepNGramOverlapScorer.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepNGramOverlapScorer.java index 31da96f0a8..50b6a6abbc 100644 --- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepNGramOverlapScorer.java +++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepNGramOverlapScorer.java @@ -1,5 +1,6 @@ package datawave.util.ssdeep; +import java.util.HashSet; import java.util.Set; /** @@ -8,17 +9,68 @@ import java.util.Set; */ public class SSDeepNGramOverlapScorer implements SSDeepHashScorer> { - NGramGenerator generator; + private final int ngramSize; - public SSDeepNGramOverlapScorer(int ngramSize, int maxRepeatedChars, int minHashSize) { - generator = new NGramGenerator(ngramSize, maxRepeatedChars, minHashSize); + public SSDeepNGramOverlapScorer(int ngramSize) { + this.ngramSize = ngramSize; } + /** + * Calculate the number of ngrams shared between a part of ssdeep hashes. Considers chunk sizes along with the single chunk / double chunk of each has as + * appropriate. + * + * @param signature1 + * the first ssdeep hash for comparison + * @param signature2 + * the second ssdeep hash for comparison + * @return the set of ngrams shared between each ssdeep hash, labeled with chunk size. + */ public Set apply(SSDeepHash signature1, SSDeepHash signature2) { - Set ngrams1 = generator.generateNgrams(signature1); - Set ngrams2 = generator.generateNgrams(signature2); + if (signature1.getChunkSize() == signature2.getChunkSize()) { + Set ngrams = new HashSet<>(); - ngrams1.retainAll(ngrams2); - return ngrams1; + ngrams.addAll(calculateOverlappingNGrams(ngramSize, signature1.getChunkSize(), signature1.getChunk(), signature2.getChunk())); + + if (signature1.hasDoubleChunk() && signature2.hasDoubleChunk()) { + ngrams.addAll(calculateOverlappingNGrams(ngramSize, signature1.getDoubleChunkSize(), signature1.getDoubleChunk(), signature2.getDoubleChunk())); + } + + return ngrams; + } else if (signature1.hasDoubleChunk() && (signature1.getDoubleChunkSize() == signature2.getChunkSize())) { + return calculateOverlappingNGrams(ngramSize, signature1.getDoubleChunkSize(), signature1.getDoubleChunk(), signature2.getChunk()); + } else if (signature2.hasDoubleChunk() && (signature1.getChunkSize() == signature2.getDoubleChunkSize())) { + return calculateOverlappingNGrams(ngramSize, signature1.getChunkSize(), signature1.getChunk(), signature2.getDoubleChunk()); + } else { + return new HashSet(); + } + } + + /** + * Calculate the number of ngrams shared amongst both chunks. Each chunk must have the same ngram size or the comparison will be invalid. For efficiency, + * the shorter chunk will drive the comparison. + * + * @param ngramSize + * the size of ngrams to compare + * @param chunkSize + * the chunk size of the ngrams we are comparing, both chunks must have the same size. + * @param chunk1 + * the first chunk to compare + * @param chunk2 + * the second chunk to compare + * @return a set of ngrams shared between each chunk, labeled with chunk size. + */ + private static Set calculateOverlappingNGrams(int ngramSize, int chunkSize, String chunk1, String chunk2) { + Set overlappingNGrams = new HashSet<>(); + final String smallChunk = chunk1.length() <= chunk2.length() ? chunk1 : chunk2; + final String largeChunk = chunk2.equals(smallChunk) ? chunk1 : chunk2; + final int ngramCount = smallChunk.length() - ngramSize; + + for (int i = 0; i <= ngramCount; i++) { + String ngram = smallChunk.substring(i, i + ngramSize); + if (largeChunk.contains(ngram)) { + overlappingNGrams.add(new NGramTuple(chunkSize, ngram)); + } + } + return overlappingNGrams; } } diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java index a5e7cf9b3b..b207324679 100644 --- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java +++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java @@ -30,7 +30,7 @@ public class SSDeepNGramOverlapScorerTest { @Test public void testCompare() { - SSDeepHashScorer> scorer = new SSDeepNGramOverlapScorer(7, 3, 6); + SSDeepHashScorer> scorer = new SSDeepNGramOverlapScorer(7); for (int i = 0; i < testData.length; i++) { SSDeepHash queryHash = SSDeepHash.parse(testData[i][0]); SSDeepHash targetHash = SSDeepHash.parse(testData[i][1]);