Merge "Select SuffixArray size at runtime."

am: 877a6678c7

Change-Id: Ib9004b35cc50111898510baef48864cbd6790a6c

Android.bp

@@ -60,6 +60,7 @@ cc_library_static {
         "diff_encoder.cc",
         "patch_writer.cc",
         "split_patch_writer.cc",
+        "suffix_array_index.cc",
     ],
     static_libs: [
         "libdivsufsort64",
@@ -106,6 +107,7 @@ cc_test {
         "extents_unittest.cc",
         "patch_writer_unittest.cc",
         "split_patch_writer_unittest.cc",
+        "suffix_array_index_unittest.cc",
         "test_utils.cc",
         "testrunner.cc",
     ],

Makefile

@@ -37,7 +37,8 @@ bsdiff_src_files := \
     bz2_compressor.cc \
     diff_encoder.cc \
     patch_writer.cc \
-    split_patch_writer.cc
+    split_patch_writer.cc \
+    suffix_array_index.cc
 
 # "bspatch" program.
 bspatch_src_files := \
@@ -58,6 +59,7 @@ bsdiff_common_unittests := \
     extents_unittest.cc \
     patch_writer_unittest.cc \
     split_patch_writer_unittest.cc \
+    suffix_array_index_unittest.cc \
     test_utils.cc \
     testrunner.cc
 

bsdiff.cc

@@ -43,61 +43,21 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05
 
 #include "bsdiff/diff_encoder.h"
 #include "bsdiff/patch_writer.h"
+#include "bsdiff/suffix_array_index.h"
 
 namespace bsdiff {
 
-static off_t matchlen(const u_char* old, off_t oldsize, const u_char* new_buf,
-                      off_t newsize) {
-	off_t i;
-
-	for(i=0;(i<oldsize)&&(i<newsize);i++)
-		if(old[i]!=new_buf[i]) break;
-
-	return i;
-}
-
-// This is a binary search of the string |new_buf| of size |newsize| (or a
-// prefix of it) in the |old| string with size |oldsize| using the suffix array
-// |I|. |st| and |en| is the start and end of the search range (inclusive).
-// Returns the length of the longest prefix found and stores the position of the
-// string found in |*pos|.
-static off_t search(saidx_t* I, const u_char* old, off_t oldsize,
-                    const u_char* new_buf, off_t newsize, off_t st, off_t en,
-                    off_t* pos) {
-	off_t x,y;
-
-	if(en-st<2) {
-		x=matchlen(old+I[st],oldsize-I[st],new_buf,newsize);
-		y=matchlen(old+I[en],oldsize-I[en],new_buf,newsize);
-
-		if(x>y) {
-			*pos=I[st];
-			return x;
-		} else {
-			*pos=I[en];
-			return y;
-		}
-	};
-
-	x=st+(en-st)/2;
-	if(memcmp(old+I[x],new_buf,std::min(oldsize-I[x],newsize))<=0) {
-		return search(I,old,oldsize,new_buf,newsize,x,en,pos);
-	} else {
-		return search(I,old,oldsize,new_buf,newsize,st,x,pos);
-	};
-}
-
 int bsdiff(const char* old_filename, const char* new_filename,
            const char* patch_filename) {
 	int fd;
-	u_char *old_buf,*new_buf;
-	off_t oldsize,newsize;
+	uint8_t *old_buf,*new_buf;
+	ssize_t oldsize,newsize;
 
 	/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
 		that we never try to malloc(0) and get a NULL pointer */
 	if(((fd=open(old_filename,O_RDONLY,0))<0) ||
 		((oldsize=lseek(fd,0,SEEK_END))==-1) ||
-		((old_buf=static_cast<u_char*>(malloc(oldsize+1)))==NULL) ||
+		((old_buf=static_cast<uint8_t*>(malloc(oldsize+1)))==NULL) ||
 		(lseek(fd,0,SEEK_SET)!=0) ||
 		(read(fd,old_buf,oldsize)!=oldsize) ||
 		(close(fd)==-1)) err(1,"%s",old_filename);
@@ -106,7 +66,7 @@ int bsdiff(const char* old_filename, const char* new_filename,
 		that we never try to malloc(0) and get a NULL pointer */
 	if(((fd=open(new_filename,O_RDONLY,0))<0) ||
 		((newsize=lseek(fd,0,SEEK_END))==-1) ||
-		((new_buf = static_cast<u_char*>(malloc(newsize+1)))==NULL) ||
+		((new_buf = static_cast<uint8_t*>(malloc(newsize+1)))==NULL) ||
 		(lseek(fd,0,SEEK_SET)!=0) ||
 		(read(fd,new_buf,newsize)!=newsize) ||
 		(close(fd)==-1)) err(1,"%s",new_filename);
@@ -122,32 +82,39 @@ int bsdiff(const char* old_filename, const char* new_filename,
 // TODO(deymo): Deprecate this version of the interface and move all callers
 // to the underlying version using PatchWriterInterface instead. This allows
 // more flexible options including different encodings.
-int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
-           off_t newsize, const char* patch_filename, saidx_t** I_cache) {
+int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf,
+           size_t newsize, const char* patch_filename,
+           SuffixArrayIndexInterface** sai_cache) {
 	BsdiffPatchWriter patch(patch_filename);
-	return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, I_cache);
+	return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, sai_cache);
 }
 
-int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
-           off_t newsize, PatchWriterInterface* patch, saidx_t** I_cache) {
-	saidx_t *I;
-	off_t scan,pos=0,len;
-	off_t lastscan,lastpos,lastoffset;
-	off_t oldscore,scsc;
-	off_t s,Sf,lenf,Sb,lenb;
-	off_t overlap,Ss,lens;
-	off_t i;
+int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf,
+           size_t newsize, PatchWriterInterface* patch,
+           SuffixArrayIndexInterface** sai_cache) {
+	size_t scsc, scan;
+	uint64_t pos=0;
+	size_t len;
+	size_t lastscan,lastpos,lastoffset;
+	uint64_t oldscore;
+	ssize_t s,Sf,lenf,Sb,lenb;
+	ssize_t overlap,Ss,lens;
+	ssize_t i;
 
-	if (I_cache && *I_cache) {
-		I = *I_cache;
+	std::unique_ptr<SuffixArrayIndexInterface> local_sai;
+	SuffixArrayIndexInterface* sai;
+
+	if (sai_cache && *sai_cache) {
+		sai = *sai_cache;
 	} else {
-		if ((I=static_cast<saidx_t*>(malloc((oldsize+1)*sizeof(saidx_t))))==NULL)
-			err(1,NULL);
+		local_sai = CreateSuffixArrayIndex(old_buf, oldsize);
+		if (!local_sai)
+			return 1;
+		sai = local_sai.get();
 
-		// Note: divsufsort() fails when the passed size is 0 and old_buf is NULL.
-		if (oldsize > 0 && divsufsort(old_buf, I, oldsize)) err(1, "divsufsort");
-		if (I_cache)
-			*I_cache = I;
+		// Transfer ownership to the caller.
+		if (sai_cache)
+			*sai_cache = local_sai.release();
 	}
 
 	/* Initialize the patch file encoder */
@@ -166,14 +133,14 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
 		 * go past that block of data. We need to track the number of
 		 * times we're stuck in the block and break out of it. */
 		int num_less_than_eight = 0;
-		off_t prev_len, prev_oldscore, prev_pos;
+		size_t prev_len;
+		uint64_t prev_pos, prev_oldscore;
 		for(scsc=scan+=len;scan<newsize;scan++) {
 			prev_len=len;
 			prev_oldscore=oldscore;
 			prev_pos=pos;
 
-			len=search(I,old_buf,oldsize,new_buf+scan,newsize-scan,
-					0,oldsize-1,&pos);
+			sai->SearchPrefix(new_buf + scan, newsize - scan, &len, &pos);
 
 			for(;scsc<scan+len;scsc++)
 			if((scsc+lastoffset<oldsize) &&
@@ -187,7 +154,7 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
 				(old_buf[scan+lastoffset] == new_buf[scan]))
 				oldscore--;
 
-			const off_t fuzz = 8;
+			const size_t fuzz = 8;
 			if (prev_len-fuzz<=len && len<=prev_len &&
 			    prev_oldscore-fuzz<=oldscore &&
 			    oldscore<=prev_oldscore &&
@@ -210,7 +177,7 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
 			lenb=0;
 			if(scan<newsize) {
 				s=0;Sb=0;
-				for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
+				for(i=1;(scan>=lastscan+i)&&(pos>=static_cast<uint64_t>(i));i++) {
 					if(old_buf[pos-i]==new_buf[scan-i]) s++;
 					if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
 				};
@@ -245,9 +212,6 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
 	if (!diff_encoder.Close())
 		errx(1, "Closing the patch file");
 
-	if (I_cache == nullptr)
-		free(I);
-
 	return 0;
 }
 

bsdiff.gyp

@@ -57,6 +57,7 @@
         'diff_encoder.cc',
         'patch_writer.cc',
         'split_patch_writer.cc',
+        'suffix_array_index.cc',
       ],
     },
     # bsdiff executable
@@ -133,6 +134,7 @@
             'extents_unittest.cc',
             'patch_writer_unittest.cc',
             'split_patch_writer_unittest.cc',
+            'suffix_array_index_unittest.cc',
             'test_utils.cc',
           ],
         },

include/bsdiff/bsdiff.h

@@ -5,18 +5,11 @@
 #ifndef _BSDIFF_BSDIFF_H_
 #define _BSDIFF_BSDIFF_H_
 
-#include <sys/types.h>
-
-#if _FILE_OFFSET_BITS == 64
-#include "divsufsort64.h"
-#define saidx_t saidx64_t
-#define divsufsort divsufsort64
-#else
-#include "divsufsort.h"
-#endif
+#include <stdint.h>
 
 #include "bsdiff/common.h"
 #include "bsdiff/patch_writer_interface.h"
+#include "bsdiff/suffix_array_index_interface.h"
 
 namespace bsdiff {
 
@@ -32,20 +25,20 @@ int bsdiff(const char* old_filename,
 // |I_cache| can be used to cache the suffix array if the same |old_buf| is used
 // repeatedly, pass nullptr if not needed.
 BSDIFF_EXPORT
-int bsdiff(const u_char* old_buf,
-           off_t oldsize,
-           const u_char* new_buf,
-           off_t newsize,
+int bsdiff(const uint8_t* old_buf,
+           size_t oldsize,
+           const uint8_t* new_buf,
+           size_t newsize,
            const char* patch_filename,
-           saidx_t** I_cache);
+           SuffixArrayIndexInterface** sai_cache);
 
 BSDIFF_EXPORT
-int bsdiff(const u_char* old_buf,
-           off_t oldsize,
-           const u_char* new_buf,
-           off_t newsize,
+int bsdiff(const uint8_t* old_buf,
+           size_t oldsize,
+           const uint8_t* new_buf,
+           size_t newsize,
            PatchWriterInterface* patch,
-           saidx_t** I_cache);
+           SuffixArrayIndexInterface** sai_cache);
 
 }  // namespace bsdiff
 

include/bsdiff/suffix_array_index_interface.h

@@ -0,0 +1,36 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_
+#define _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_
+
+// The SuffixArrayIndexInterface encapsulates a search index based on a
+// suffix-array with a common string search interface. The implementations of
+// this index can vary on technical details, such as the size of the internal
+// suffix array elements, which are not visible in this interface.
+
+#include <stdint.h>
+
+namespace bsdiff {
+
+class SuffixArrayIndexInterface {
+ public:
+  virtual ~SuffixArrayIndexInterface() = default;
+
+  // Search in the index the longest prefix of the string |target| of length
+  // |length|. The length of the longest prefix found, which could be 0, is
+  // stored in |out_length| and a position in the source text where this prefix
+  // was found is store in |out_pos|.
+  virtual void SearchPrefix(const uint8_t* target,
+                            size_t length,
+                            size_t* out_length,
+                            uint64_t* out_pos) const = 0;
+
+ protected:
+  SuffixArrayIndexInterface() = default;
+};
+
+}  // namespace bsdiff
+
+#endif  // _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_

suffix_array_index.cc

@@ -0,0 +1,174 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "bsdiff/suffix_array_index.h"
+
+#include <limits>
+#include <vector>
+
+#include <divsufsort.h>
+#include <divsufsort64.h>
+
+#include "bsdiff/logging.h"
+
+using std::endl;
+
+namespace {
+
+// libdivsufsort C++ overloaded functions used to allow calling the right
+// implementation based on the pointer size.
+int CallDivSufSort(const uint8_t* text, saidx_t* sa, size_t n) {
+  return divsufsort(text, sa, n);
+}
+int CallDivSufSort(const uint8_t* text, saidx64_t* sa, size_t n) {
+  return divsufsort64(text, sa, n);
+}
+
+saidx_t CallSaSearch(const uint8_t* text,
+                     size_t text_size,
+                     const uint8_t* pattern,
+                     size_t pattern_size,
+                     const saidx_t* sa,
+                     size_t sa_size,
+                     saidx_t* left) {
+  return sa_search(text, text_size, pattern, pattern_size, sa, sa_size, left);
+}
+saidx64_t CallSaSearch(const uint8_t* text,
+                       size_t text_size,
+                       const uint8_t* pattern,
+                       size_t pattern_size,
+                       const saidx64_t* sa,
+                       size_t sa_size,
+                       saidx64_t* left) {
+  return sa_search64(text, text_size, pattern, pattern_size, sa, sa_size, left);
+}
+
+}  // namespace
+
+namespace bsdiff {
+
+// The SAIDX template type must be either saidx_t or saidx64_t, which will
+// depend on the maximum size of the input text needed.
+template <typename SAIDX>
+class SuffixArrayIndex : public SuffixArrayIndexInterface {
+ public:
+  SuffixArrayIndex() = default;
+
+  // Initialize and construct the suffix array of the |text| string of length
+  // |n|. The memory pointed by |text| must be kept alive. Returns whether the
+  // construction succeeded.
+  bool Init(const uint8_t* text, size_t n);
+
+  // SuffixArrayIndexInterface overrides.
+  void SearchPrefix(const uint8_t* target,
+                    size_t length,
+                    size_t* out_length,
+                    uint64_t* out_pos) const override;
+
+ private:
+  const uint8_t* text_{nullptr};  // Owned by the caller.
+  size_t n_{0};
+
+  std::vector<SAIDX> sa_;
+};
+
+template <typename SAIDX>
+bool SuffixArrayIndex<SAIDX>::Init(const uint8_t* text, size_t n) {
+  if (!sa_.empty()) {
+    // Already initialized.
+    LOG(ERROR) << "SuffixArray already initialized" << endl;
+    return false;
+  }
+  if (static_cast<uint64_t>(n) >
+      static_cast<uint64_t>(std::numeric_limits<SAIDX>::max())) {
+    LOG(ERROR) << "Input too big (" << n << ") for this implementation" << endl;
+    return false;
+  }
+  text_ = text;
+  n_ = n;
+  sa_.resize(n + 1);
+
+  if (n > 0 && CallDivSufSort(text_, sa_.data(), n) != 0) {
+    LOG(ERROR) << "divsufsrot() failed" << endl;
+    return false;
+  }
+
+  return true;
+}
+
+template <typename SAIDX>
+void SuffixArrayIndex<SAIDX>::SearchPrefix(const uint8_t* target,
+                                           size_t length,
+                                           size_t* out_length,
+                                           uint64_t* out_pos) const {
+  SAIDX suf_left;
+  SAIDX count =
+      CallSaSearch(text_, n_, target, length, sa_.data(), n_, &suf_left);
+  if (count > 0) {
+    // This is the simple case where we found the whole |target| string was
+    // found.
+    *out_pos = sa_[suf_left];
+    *out_length = length;
+    return;
+  }
+  // In this case, |suf_left| points to the first suffix array position such
+  // that the suffix at that position is lexicographically larger than |target|.
+  // We only need to check whether the previous entry or the current entry is a
+  // longer match.
+  size_t prev_suffix_len = 0;
+  if (suf_left > 0) {
+    const size_t prev_max_len =
+        std::min(n_ - static_cast<size_t>(sa_[suf_left - 1]), length);
+    const uint8_t* prev_suffix = text_ + sa_[suf_left - 1];
+    prev_suffix_len =
+        std::mismatch(target, target + prev_max_len, prev_suffix).first -
+        target;
+  }
+
+  size_t next_suffix_len = 0;
+  if (static_cast<size_t>(suf_left) < n_) {
+    const uint8_t* next_suffix = text_ + sa_[suf_left];
+    const size_t next_max_len =
+        std::min(n_ - static_cast<size_t>(sa_[suf_left]), length);
+    next_suffix_len =
+        std::mismatch(target, target + next_max_len, next_suffix).first -
+        target;
+  }
+
+  *out_length = std::max(next_suffix_len, prev_suffix_len);
+  if (!*out_length) {
+    *out_pos = 0;
+  } else if (next_suffix_len >= prev_suffix_len) {
+    *out_pos = sa_[suf_left];
+  } else {
+    *out_pos = sa_[suf_left - 1];
+  }
+}
+
+std::unique_ptr<SuffixArrayIndexInterface> CreateSuffixArrayIndex(
+    const uint8_t* text,
+    size_t n) {
+  // The maximum supported size when using the suffix array based on the 32-bit
+  // saidx_t type. We limit this to something a bit smaller (16 bytes smaller)
+  // than the maximum representable number so references like "n + 1" are don't
+  // overflow.
+  const size_t kMaxSaidxSize = std::numeric_limits<saidx_t>::max() - 16;
+  std::unique_ptr<SuffixArrayIndexInterface> ret;
+
+  if (n > kMaxSaidxSize) {
+    SuffixArrayIndex<saidx64_t>* sa_ptr = new SuffixArrayIndex<saidx64_t>();
+    ret.reset(sa_ptr);
+    if (!sa_ptr->Init(text, n))
+      return nullptr;
+  } else {
+    SuffixArrayIndex<saidx_t>* sa_ptr = new SuffixArrayIndex<saidx_t>();
+    ret.reset(sa_ptr);
+    if (!sa_ptr->Init(text, n))
+      return nullptr;
+  }
+  return ret;
+}
+
+
+}  // namespace bsdiff

suffix_array_index.h

@@ -0,0 +1,22 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef _BSDIFF_SUFFIX_ARRAY_INDEX_H_
+#define _BSDIFF_SUFFIX_ARRAY_INDEX_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "bsdiff/suffix_array_index_interface.h"
+
+namespace bsdiff {
+
+std::unique_ptr<SuffixArrayIndexInterface> CreateSuffixArrayIndex(
+    const uint8_t* text,
+    size_t n);
+
+}  // namespace bsdiff
+
+#endif  // _BSDIFF_SUFFIX_ARRAY_INDEX_H_

suffix_array_index_unittest.cc

@@ -0,0 +1,79 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "bsdiff/suffix_array_index.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace bsdiff {
+
+class SuffixArrayIndexTest : public ::testing::Test {
+ protected:
+  void InitSA(const std::string& text) {
+    text_.resize(text.size());
+    std::copy(text.begin(), text.end(), text_.data());
+    sai_ = CreateSuffixArrayIndex(text_.data(), text_.size());
+  }
+
+  void SearchPrefix(const std::string& pattern,
+                    size_t* out_length,
+                    uint64_t* out_pos) {
+    sai_->SearchPrefix(reinterpret_cast<const uint8_t*>(pattern.data()),
+                       pattern.size(), out_length, out_pos);
+    // Check that the returned values are indeed a valid prefix.
+    EXPECT_LE(*out_length, pattern.size());
+    ASSERT_LT(*out_pos, text_.size());
+    ASSERT_LE(*out_pos + *out_length, text_.size());
+    EXPECT_EQ(0, memcmp(text_.data() + *out_pos, pattern.data(), *out_length));
+  }
+
+  std::vector<uint8_t> text_;
+  std::unique_ptr<SuffixArrayIndexInterface> sai_;
+};
+
+// Test that the suffix array index can be initialized with an example text.
+TEST_F(SuffixArrayIndexTest, MississippiTest) {
+  InitSA("mississippi");
+}
+
+TEST_F(SuffixArrayIndexTest, EmptySuffixArrayTest) {
+  InitSA("");
+}
+
+// Test various corner cases while searching for prefixes.
+TEST_F(SuffixArrayIndexTest, SearchPrefixTest) {
+  InitSA("abc1_abc2_abc3_ab_abcd");
+
+  uint64_t pos;
+  size_t length;
+  // The string is not found at all.
+  SearchPrefix("zzz", &length, &pos);  // lexicographically highest.
+  EXPECT_EQ(0U, length);
+
+  SearchPrefix("   ", &length, &pos);  // lexicographically lowest.
+  EXPECT_EQ(0U, length);
+
+  // The exact pattern is found multiple times.
+  SearchPrefix("abc", &length, &pos);
+  EXPECT_EQ(3U, length);
+
+  // The exact pattern is found only one time, at the end of the string.
+  SearchPrefix("abcd", &length, &pos);
+  EXPECT_EQ(4U, length);
+  EXPECT_EQ(18U, pos);
+
+  // The string is not found, but a prefix is found.
+  SearchPrefix("abcW", &length, &pos);
+  EXPECT_EQ(3U, length);
+}
+
+}  // namespace bsdiff