diff --git a/Android.bp b/Android.bp index 9907383..3d042d8 100644 --- a/Android.bp +++ b/Android.bp @@ -60,6 +60,7 @@ cc_library_static { "diff_encoder.cc", "patch_writer.cc", "split_patch_writer.cc", + "suffix_array_index.cc", ], static_libs: [ "libdivsufsort64", @@ -106,6 +107,7 @@ cc_test { "extents_unittest.cc", "patch_writer_unittest.cc", "split_patch_writer_unittest.cc", + "suffix_array_index_unittest.cc", "test_utils.cc", "testrunner.cc", ], diff --git a/Makefile b/Makefile index 0a881a7..c3ecc85 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,8 @@ bsdiff_src_files := \ bz2_compressor.cc \ diff_encoder.cc \ patch_writer.cc \ - split_patch_writer.cc + split_patch_writer.cc \ + suffix_array_index.cc # "bspatch" program. bspatch_src_files := \ @@ -58,6 +59,7 @@ bsdiff_common_unittests := \ extents_unittest.cc \ patch_writer_unittest.cc \ split_patch_writer_unittest.cc \ + suffix_array_index_unittest.cc \ test_utils.cc \ testrunner.cc diff --git a/bsdiff.cc b/bsdiff.cc index 8118e36..ec9e81b 100644 --- a/bsdiff.cc +++ b/bsdiff.cc @@ -43,61 +43,21 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 #include "bsdiff/diff_encoder.h" #include "bsdiff/patch_writer.h" +#include "bsdiff/suffix_array_index.h" namespace bsdiff { -static off_t matchlen(const u_char* old, off_t oldsize, const u_char* new_buf, - off_t newsize) { - off_t i; - - for(i=0;(iy) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new_buf,std::min(oldsize-I[x],newsize))<=0) { - return search(I,old,oldsize,new_buf,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new_buf,newsize,st,x,pos); - }; -} - int bsdiff(const char* old_filename, const char* new_filename, const char* patch_filename) { int fd; - u_char *old_buf,*new_buf; - off_t oldsize,newsize; + uint8_t *old_buf,*new_buf; + ssize_t oldsize,newsize; /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure that we never try to malloc(0) and get a NULL pointer */ if(((fd=open(old_filename,O_RDONLY,0))<0) || ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old_buf=static_cast(malloc(oldsize+1)))==NULL) || + ((old_buf=static_cast(malloc(oldsize+1)))==NULL) || (lseek(fd,0,SEEK_SET)!=0) || (read(fd,old_buf,oldsize)!=oldsize) || (close(fd)==-1)) err(1,"%s",old_filename); @@ -106,7 +66,7 @@ int bsdiff(const char* old_filename, const char* new_filename, that we never try to malloc(0) and get a NULL pointer */ if(((fd=open(new_filename,O_RDONLY,0))<0) || ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new_buf = static_cast(malloc(newsize+1)))==NULL) || + ((new_buf = static_cast(malloc(newsize+1)))==NULL) || (lseek(fd,0,SEEK_SET)!=0) || (read(fd,new_buf,newsize)!=newsize) || (close(fd)==-1)) err(1,"%s",new_filename); @@ -122,32 +82,39 @@ int bsdiff(const char* old_filename, const char* new_filename, // TODO(deymo): Deprecate this version of the interface and move all callers // to the underlying version using PatchWriterInterface instead. This allows // more flexible options including different encodings. -int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf, - off_t newsize, const char* patch_filename, saidx_t** I_cache) { +int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf, + size_t newsize, const char* patch_filename, + SuffixArrayIndexInterface** sai_cache) { BsdiffPatchWriter patch(patch_filename); - return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, I_cache); + return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, sai_cache); } -int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf, - off_t newsize, PatchWriterInterface* patch, saidx_t** I_cache) { - saidx_t *I; - off_t scan,pos=0,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; +int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf, + size_t newsize, PatchWriterInterface* patch, + SuffixArrayIndexInterface** sai_cache) { + size_t scsc, scan; + uint64_t pos=0; + size_t len; + size_t lastscan,lastpos,lastoffset; + uint64_t oldscore; + ssize_t s,Sf,lenf,Sb,lenb; + ssize_t overlap,Ss,lens; + ssize_t i; - if (I_cache && *I_cache) { - I = *I_cache; + std::unique_ptr local_sai; + SuffixArrayIndexInterface* sai; + + if (sai_cache && *sai_cache) { + sai = *sai_cache; } else { - if ((I=static_cast(malloc((oldsize+1)*sizeof(saidx_t))))==NULL) - err(1,NULL); + local_sai = CreateSuffixArrayIndex(old_buf, oldsize); + if (!local_sai) + return 1; + sai = local_sai.get(); - // Note: divsufsort() fails when the passed size is 0 and old_buf is NULL. - if (oldsize > 0 && divsufsort(old_buf, I, oldsize)) err(1, "divsufsort"); - if (I_cache) - *I_cache = I; + // Transfer ownership to the caller. + if (sai_cache) + *sai_cache = local_sai.release(); } /* Initialize the patch file encoder */ @@ -166,14 +133,14 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf, * go past that block of data. We need to track the number of * times we're stuck in the block and break out of it. */ int num_less_than_eight = 0; - off_t prev_len, prev_oldscore, prev_pos; + size_t prev_len; + uint64_t prev_pos, prev_oldscore; for(scsc=scan+=len;scanSearchPrefix(new_buf + scan, newsize - scan, &len, &pos); for(;scsc=lastscan+i)&&(pos>=i);i++) { + for(i=1;(scan>=lastscan+i)&&(pos>=static_cast(i));i++) { if(old_buf[pos-i]==new_buf[scan-i]) s++; if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; }; @@ -245,9 +212,6 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf, if (!diff_encoder.Close()) errx(1, "Closing the patch file"); - if (I_cache == nullptr) - free(I); - return 0; } diff --git a/bsdiff.gyp b/bsdiff.gyp index 87f0c2b..cc81588 100644 --- a/bsdiff.gyp +++ b/bsdiff.gyp @@ -57,6 +57,7 @@ 'diff_encoder.cc', 'patch_writer.cc', 'split_patch_writer.cc', + 'suffix_array_index.cc', ], }, # bsdiff executable @@ -133,6 +134,7 @@ 'extents_unittest.cc', 'patch_writer_unittest.cc', 'split_patch_writer_unittest.cc', + 'suffix_array_index_unittest.cc', 'test_utils.cc', ], }, diff --git a/include/bsdiff/bsdiff.h b/include/bsdiff/bsdiff.h index 5d710f5..e23b690 100644 --- a/include/bsdiff/bsdiff.h +++ b/include/bsdiff/bsdiff.h @@ -5,18 +5,11 @@ #ifndef _BSDIFF_BSDIFF_H_ #define _BSDIFF_BSDIFF_H_ -#include - -#if _FILE_OFFSET_BITS == 64 -#include "divsufsort64.h" -#define saidx_t saidx64_t -#define divsufsort divsufsort64 -#else -#include "divsufsort.h" -#endif +#include #include "bsdiff/common.h" #include "bsdiff/patch_writer_interface.h" +#include "bsdiff/suffix_array_index_interface.h" namespace bsdiff { @@ -32,20 +25,20 @@ int bsdiff(const char* old_filename, // |I_cache| can be used to cache the suffix array if the same |old_buf| is used // repeatedly, pass nullptr if not needed. BSDIFF_EXPORT -int bsdiff(const u_char* old_buf, - off_t oldsize, - const u_char* new_buf, - off_t newsize, +int bsdiff(const uint8_t* old_buf, + size_t oldsize, + const uint8_t* new_buf, + size_t newsize, const char* patch_filename, - saidx_t** I_cache); + SuffixArrayIndexInterface** sai_cache); BSDIFF_EXPORT -int bsdiff(const u_char* old_buf, - off_t oldsize, - const u_char* new_buf, - off_t newsize, +int bsdiff(const uint8_t* old_buf, + size_t oldsize, + const uint8_t* new_buf, + size_t newsize, PatchWriterInterface* patch, - saidx_t** I_cache); + SuffixArrayIndexInterface** sai_cache); } // namespace bsdiff diff --git a/include/bsdiff/suffix_array_index_interface.h b/include/bsdiff/suffix_array_index_interface.h new file mode 100644 index 0000000..0292883 --- /dev/null +++ b/include/bsdiff/suffix_array_index_interface.h @@ -0,0 +1,36 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_ +#define _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_ + +// The SuffixArrayIndexInterface encapsulates a search index based on a +// suffix-array with a common string search interface. The implementations of +// this index can vary on technical details, such as the size of the internal +// suffix array elements, which are not visible in this interface. + +#include + +namespace bsdiff { + +class SuffixArrayIndexInterface { + public: + virtual ~SuffixArrayIndexInterface() = default; + + // Search in the index the longest prefix of the string |target| of length + // |length|. The length of the longest prefix found, which could be 0, is + // stored in |out_length| and a position in the source text where this prefix + // was found is store in |out_pos|. + virtual void SearchPrefix(const uint8_t* target, + size_t length, + size_t* out_length, + uint64_t* out_pos) const = 0; + + protected: + SuffixArrayIndexInterface() = default; +}; + +} // namespace bsdiff + +#endif // _BSDIFF_SUFFIX_ARRAY_INDEX_INTERFACE_H_ diff --git a/suffix_array_index.cc b/suffix_array_index.cc new file mode 100644 index 0000000..27d6cb3 --- /dev/null +++ b/suffix_array_index.cc @@ -0,0 +1,174 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/suffix_array_index.h" + +#include +#include + +#include +#include + +#include "bsdiff/logging.h" + +using std::endl; + +namespace { + +// libdivsufsort C++ overloaded functions used to allow calling the right +// implementation based on the pointer size. +int CallDivSufSort(const uint8_t* text, saidx_t* sa, size_t n) { + return divsufsort(text, sa, n); +} +int CallDivSufSort(const uint8_t* text, saidx64_t* sa, size_t n) { + return divsufsort64(text, sa, n); +} + +saidx_t CallSaSearch(const uint8_t* text, + size_t text_size, + const uint8_t* pattern, + size_t pattern_size, + const saidx_t* sa, + size_t sa_size, + saidx_t* left) { + return sa_search(text, text_size, pattern, pattern_size, sa, sa_size, left); +} +saidx64_t CallSaSearch(const uint8_t* text, + size_t text_size, + const uint8_t* pattern, + size_t pattern_size, + const saidx64_t* sa, + size_t sa_size, + saidx64_t* left) { + return sa_search64(text, text_size, pattern, pattern_size, sa, sa_size, left); +} + +} // namespace + +namespace bsdiff { + +// The SAIDX template type must be either saidx_t or saidx64_t, which will +// depend on the maximum size of the input text needed. +template +class SuffixArrayIndex : public SuffixArrayIndexInterface { + public: + SuffixArrayIndex() = default; + + // Initialize and construct the suffix array of the |text| string of length + // |n|. The memory pointed by |text| must be kept alive. Returns whether the + // construction succeeded. + bool Init(const uint8_t* text, size_t n); + + // SuffixArrayIndexInterface overrides. + void SearchPrefix(const uint8_t* target, + size_t length, + size_t* out_length, + uint64_t* out_pos) const override; + + private: + const uint8_t* text_{nullptr}; // Owned by the caller. + size_t n_{0}; + + std::vector sa_; +}; + +template +bool SuffixArrayIndex::Init(const uint8_t* text, size_t n) { + if (!sa_.empty()) { + // Already initialized. + LOG(ERROR) << "SuffixArray already initialized" << endl; + return false; + } + if (static_cast(n) > + static_cast(std::numeric_limits::max())) { + LOG(ERROR) << "Input too big (" << n << ") for this implementation" << endl; + return false; + } + text_ = text; + n_ = n; + sa_.resize(n + 1); + + if (n > 0 && CallDivSufSort(text_, sa_.data(), n) != 0) { + LOG(ERROR) << "divsufsrot() failed" << endl; + return false; + } + + return true; +} + +template +void SuffixArrayIndex::SearchPrefix(const uint8_t* target, + size_t length, + size_t* out_length, + uint64_t* out_pos) const { + SAIDX suf_left; + SAIDX count = + CallSaSearch(text_, n_, target, length, sa_.data(), n_, &suf_left); + if (count > 0) { + // This is the simple case where we found the whole |target| string was + // found. + *out_pos = sa_[suf_left]; + *out_length = length; + return; + } + // In this case, |suf_left| points to the first suffix array position such + // that the suffix at that position is lexicographically larger than |target|. + // We only need to check whether the previous entry or the current entry is a + // longer match. + size_t prev_suffix_len = 0; + if (suf_left > 0) { + const size_t prev_max_len = + std::min(n_ - static_cast(sa_[suf_left - 1]), length); + const uint8_t* prev_suffix = text_ + sa_[suf_left - 1]; + prev_suffix_len = + std::mismatch(target, target + prev_max_len, prev_suffix).first - + target; + } + + size_t next_suffix_len = 0; + if (static_cast(suf_left) < n_) { + const uint8_t* next_suffix = text_ + sa_[suf_left]; + const size_t next_max_len = + std::min(n_ - static_cast(sa_[suf_left]), length); + next_suffix_len = + std::mismatch(target, target + next_max_len, next_suffix).first - + target; + } + + *out_length = std::max(next_suffix_len, prev_suffix_len); + if (!*out_length) { + *out_pos = 0; + } else if (next_suffix_len >= prev_suffix_len) { + *out_pos = sa_[suf_left]; + } else { + *out_pos = sa_[suf_left - 1]; + } +} + +std::unique_ptr CreateSuffixArrayIndex( + const uint8_t* text, + size_t n) { + // The maximum supported size when using the suffix array based on the 32-bit + // saidx_t type. We limit this to something a bit smaller (16 bytes smaller) + // than the maximum representable number so references like "n + 1" are don't + // overflow. + const size_t kMaxSaidxSize = std::numeric_limits::max() - 16; + std::unique_ptr ret; + + if (n > kMaxSaidxSize) { + SuffixArrayIndex* sa_ptr = new SuffixArrayIndex(); + ret.reset(sa_ptr); + if (!sa_ptr->Init(text, n)) + return nullptr; + } else { + SuffixArrayIndex* sa_ptr = new SuffixArrayIndex(); + ret.reset(sa_ptr); + if (!sa_ptr->Init(text, n)) + return nullptr; + } + return ret; +} + + +} // namespace bsdiff diff --git a/suffix_array_index.h b/suffix_array_index.h new file mode 100644 index 0000000..acf4f19 --- /dev/null +++ b/suffix_array_index.h @@ -0,0 +1,22 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef _BSDIFF_SUFFIX_ARRAY_INDEX_H_ +#define _BSDIFF_SUFFIX_ARRAY_INDEX_H_ + +#include + +#include + +#include "bsdiff/suffix_array_index_interface.h" + +namespace bsdiff { + +std::unique_ptr CreateSuffixArrayIndex( + const uint8_t* text, + size_t n); + +} // namespace bsdiff + +#endif // _BSDIFF_SUFFIX_ARRAY_INDEX_H_ diff --git a/suffix_array_index_unittest.cc b/suffix_array_index_unittest.cc new file mode 100644 index 0000000..50563d4 --- /dev/null +++ b/suffix_array_index_unittest.cc @@ -0,0 +1,79 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/suffix_array_index.h" + +#include + +#include +#include +#include +#include + +#include +#include + +namespace bsdiff { + +class SuffixArrayIndexTest : public ::testing::Test { + protected: + void InitSA(const std::string& text) { + text_.resize(text.size()); + std::copy(text.begin(), text.end(), text_.data()); + sai_ = CreateSuffixArrayIndex(text_.data(), text_.size()); + } + + void SearchPrefix(const std::string& pattern, + size_t* out_length, + uint64_t* out_pos) { + sai_->SearchPrefix(reinterpret_cast(pattern.data()), + pattern.size(), out_length, out_pos); + // Check that the returned values are indeed a valid prefix. + EXPECT_LE(*out_length, pattern.size()); + ASSERT_LT(*out_pos, text_.size()); + ASSERT_LE(*out_pos + *out_length, text_.size()); + EXPECT_EQ(0, memcmp(text_.data() + *out_pos, pattern.data(), *out_length)); + } + + std::vector text_; + std::unique_ptr sai_; +}; + +// Test that the suffix array index can be initialized with an example text. +TEST_F(SuffixArrayIndexTest, MississippiTest) { + InitSA("mississippi"); +} + +TEST_F(SuffixArrayIndexTest, EmptySuffixArrayTest) { + InitSA(""); +} + +// Test various corner cases while searching for prefixes. +TEST_F(SuffixArrayIndexTest, SearchPrefixTest) { + InitSA("abc1_abc2_abc3_ab_abcd"); + + uint64_t pos; + size_t length; + // The string is not found at all. + SearchPrefix("zzz", &length, &pos); // lexicographically highest. + EXPECT_EQ(0U, length); + + SearchPrefix(" ", &length, &pos); // lexicographically lowest. + EXPECT_EQ(0U, length); + + // The exact pattern is found multiple times. + SearchPrefix("abc", &length, &pos); + EXPECT_EQ(3U, length); + + // The exact pattern is found only one time, at the end of the string. + SearchPrefix("abcd", &length, &pos); + EXPECT_EQ(4U, length); + EXPECT_EQ(18U, pos); + + // The string is not found, but a prefix is found. + SearchPrefix("abcW", &length, &pos); + EXPECT_EQ(3U, length); +} + +} // namespace bsdiff