diff --git a/brotli_compressor.cc b/brotli_compressor.cc index 8238650..6adc3e8 100644 --- a/brotli_compressor.cc +++ b/brotli_compressor.cc @@ -12,7 +12,7 @@ namespace { // TODO(xunchang) set brotli compression parameters based on input options. const size_t kBufferSize = 1024 * 1024; const uint32_t kBrotliQuality = 11; -const uint32_t kBrotliLgwin = 24; +const uint32_t kBrotliLgwin = 20; } // namespace diff --git a/brotli_compressor.h b/brotli_compressor.h index 3779823..b6d698a 100644 --- a/brotli_compressor.h +++ b/brotli_compressor.h @@ -13,6 +13,7 @@ #include "bsdiff/compressor_buffer.h" #include "bsdiff/compressor_interface.h" +#include "bsdiff/constants.h" namespace bsdiff { @@ -25,6 +26,7 @@ class BrotliCompressor : public CompressorInterface { bool Write(const uint8_t* buf, size_t size) override; bool Finish() override; const std::vector& GetCompressedData() override; + CompressorType Type() override { return CompressorType::kBrotli; } private: BrotliEncoderState* brotli_encoder_state_; diff --git a/bsdiff.cc b/bsdiff.cc index e5d1e7c..ffb3bbf 100644 --- a/bsdiff.cc +++ b/bsdiff.cc @@ -49,7 +49,7 @@ namespace bsdiff { int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf, size_t newsize, const char* patch_filename, SuffixArrayIndexInterface** sai_cache) { - BsdiffPatchWriter patch(patch_filename, CompressorType::kBZ2); + BsdiffPatchWriter patch(patch_filename, BsdiffFormat::kLegacy); return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, sai_cache); } diff --git a/bz2_compressor.h b/bz2_compressor.h index b6f4dfe..5e2e9f6 100644 --- a/bz2_compressor.h +++ b/bz2_compressor.h @@ -29,6 +29,7 @@ class BZ2Compressor : public CompressorInterface { bool Write(const uint8_t* buf, size_t size) override; bool Finish() override; const std::vector& GetCompressedData() override; + CompressorType Type() override { return CompressorType::kBZ2; } private: // The low-level bzip2 stream. diff --git a/compressor_interface.cc b/compressor_interface.cc index 260a442..9ceb6bc 100644 --- a/compressor_interface.cc +++ b/compressor_interface.cc @@ -2,19 +2,25 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include +#include "bsdiff/compressor_interface.h" -#include -#include +#include "bsdiff/brotli_compressor.h" +#include "bsdiff/bz2_compressor.h" +#include "bsdiff/logging.h" namespace bsdiff { std::unique_ptr CreateCompressor(CompressorType type) { - if (type == CompressorType::kBrotli) { - return std::unique_ptr(new BrotliCompressor()); + switch (type) { + case CompressorType::kBZ2: + return std::unique_ptr(new BZ2Compressor()); + case CompressorType::kBrotli: + return std::unique_ptr(new BrotliCompressor()); + default: + LOG(ERROR) << "unsupported compressor type: " + << static_cast(type) << std::endl; + return nullptr; } - // Use BZ2 as a default compressor. - return std::unique_ptr(new BZ2Compressor()); } } // namespace bsdiff diff --git a/compressor_interface.h b/compressor_interface.h index 53a33c8..46c5b72 100644 --- a/compressor_interface.h +++ b/compressor_interface.h @@ -28,6 +28,9 @@ class CompressorInterface { // Return the compressed data. This method must be only called after Finish(). virtual const std::vector& GetCompressedData() = 0; + // Return the type of the current compressor. + virtual CompressorType Type() = 0; + protected: CompressorInterface() = default; }; diff --git a/constants.h b/constants.h index fd0da26..5236637 100644 --- a/constants.h +++ b/constants.h @@ -9,15 +9,19 @@ namespace bsdiff { -enum class CompressorType { - kBrotli, - kBZ2, +enum class CompressorType : uint8_t { + kNoCompression = 0, + kBZ2 = 1, + kBrotli = 2, }; // The header of the upstream's "BSDIFF40" format using BZ2 as compressor. -constexpr uint8_t kMagicHeader[] = "BSDIFF40"; +constexpr uint8_t kLegacyMagicHeader[] = "BSDIFF40"; + +// The header of the new BSDF2 format. This format supports both bz2 and +// brotli compressor. +constexpr uint8_t kBSDF2MagicHeader[] = "BSDF2"; } // namespace bsdiff - #endif // _BSDIFF_CONSTANTS_H_ \ No newline at end of file diff --git a/decompressor_interface.cc b/decompressor_interface.cc index 69c611f..f5fdb2d 100644 --- a/decompressor_interface.cc +++ b/decompressor_interface.cc @@ -6,15 +6,21 @@ #include "bsdiff/brotli_decompressor.h" #include "bsdiff/bz2_decompressor.h" +#include "bsdiff/logging.h" namespace bsdiff { std::unique_ptr CreateDecompressor(CompressorType type) { - if (type == CompressorType::kBrotli) { - return std::unique_ptr(new BrotliDecompressor()); + switch (type) { + case CompressorType::kBZ2: + return std::unique_ptr(new BZ2Decompressor()); + case CompressorType::kBrotli: + return std::unique_ptr(new BrotliDecompressor()); + default: + LOG(ERROR) << "unsupported compressor type: " + << static_cast(type) << std::endl; + return nullptr; } - // Use BZ2 as a default decompressor. - return std::unique_ptr(new BZ2Decompressor()); } } // namespace bsdiff \ No newline at end of file diff --git a/include/bsdiff/patch_writer_interface.h b/include/bsdiff/patch_writer_interface.h index 0f9adfb..31fc3d8 100644 --- a/include/bsdiff/patch_writer_interface.h +++ b/include/bsdiff/patch_writer_interface.h @@ -12,6 +12,11 @@ namespace bsdiff { +enum class BsdiffFormat { + kLegacy, + kBsdf2, +}; + class PatchWriterInterface { public: virtual ~PatchWriterInterface() = default; diff --git a/patch_reader.cc b/patch_reader.cc index 5fd0ac6..80a61f8 100644 --- a/patch_reader.cc +++ b/patch_reader.cc @@ -20,8 +20,8 @@ using std::endl; namespace bsdiff { bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) { - // File format: - // 0 8 "BSDIFF40" + // File format: + // 0 8 magic header // 8 8 X // 16 8 Y // 24 8 new_file_size @@ -32,8 +32,38 @@ bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) { // from oldfile to x bytes from the diff block; copy y bytes from the // extra block; seek forwards in oldfile by z bytes". + if (patch_size < 32) { + LOG(ERROR) << "Too small to be a bspatch." << endl; + return false; + } // Check for appropriate magic. - if (memcmp(patch_data, kMagicHeader, 8) != 0) { + std::vector compression_type; + if (memcmp(patch_data, kLegacyMagicHeader, 8) == 0) { + // The magic header is "BSDIFF40" for legacy format. + compression_type = {CompressorType::kBZ2, CompressorType::kBZ2, + CompressorType::kBZ2}; + } else if (memcmp(patch_data, kBSDF2MagicHeader, 5) == 0) { + // The magic header for BSDF2 format: + // 0 5 BSDF2 + // 5 1 compressed type for control stream + // 6 1 compressed type for diff stream + // 7 1 compressed type for extra stream + for (size_t i = 5; i < 8; i++) { + uint8_t type = patch_data[i]; + switch (type) { + case static_cast(CompressorType::kBZ2): + compression_type.push_back(CompressorType::kBZ2); + break; + case static_cast(CompressorType::kBrotli): + compression_type.push_back(CompressorType::kBrotli); + break; + default: + LOG(ERROR) << "Unsupported compression type: " + << static_cast(type) << endl; + return false; + } + } + } else { LOG(ERROR) << "Not a bsdiff patch." << endl; return false; } @@ -47,18 +77,20 @@ bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) { LOG(ERROR) << "Corrupt patch. ctrl_len: " << ctrl_len << ", data_len: " << diff_len << ", new_file_size: " << signed_newsize - << ", patch_size: " << patch_size; + << ", patch_size: " << patch_size << endl; return false; } new_file_size_ = signed_newsize; - // TODO(xunchang) set the correct decompressor based on the info in the - // header. - ctrl_stream_ = CreateDecompressor(CompressorType::kBZ2); - diff_stream_ = CreateDecompressor(CompressorType::kBZ2); - extra_stream_ = CreateDecompressor(CompressorType::kBZ2); + ctrl_stream_ = CreateDecompressor(compression_type[0]); + diff_stream_ = CreateDecompressor(compression_type[1]); + extra_stream_ = CreateDecompressor(compression_type[2]); + if (!(ctrl_stream_ && diff_stream_ && extra_stream_)) { + LOG(ERROR) << "uninitialized decompressor stream" << endl; + return false; + } - int64_t offset = 32; + size_t offset = 32; if (!ctrl_stream_->SetInputData(const_cast(patch_data) + offset, ctrl_len)) { LOG(ERROR) << "Failed to init ctrl stream, ctrl_len: " << ctrl_len << endl; @@ -122,17 +154,17 @@ bool BsdiffPatchReader::ReadExtraStream(uint8_t* buf, size_t size) { bool BsdiffPatchReader::Finish() { if (!ctrl_stream_->Close()) { - LOG(ERROR) << "Failed to close the control stream"; + LOG(ERROR) << "Failed to close the control stream" << endl; return false; } if (!diff_stream_->Close()) { - LOG(ERROR) << "Failed to close the diff stream"; + LOG(ERROR) << "Failed to close the diff stream" << endl; return false; } if (!extra_stream_->Close()) { - LOG(ERROR) << "Failed to close the extra stream"; + LOG(ERROR) << "Failed to close the extra stream" << endl; return false; } return true; diff --git a/patch_reader_unittest.cc b/patch_reader_unittest.cc index 85fae28..61b63b7 100644 --- a/patch_reader_unittest.cc +++ b/patch_reader_unittest.cc @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "bsdiff/patch_reader.h" + #include #include @@ -10,8 +12,8 @@ #include +#include "bsdiff/brotli_compressor.h" #include "bsdiff/bz2_compressor.h" -#include "bsdiff/patch_reader.h" #include "bsdiff/utils.h" namespace { @@ -28,69 +30,116 @@ void EncodeInt64(int64_t x, uint8_t* buf) { namespace bsdiff { -TEST(PatchReaderTest, BZ2PatchReaderSmoke) { - BZ2Compressor ctrl_stream; - BZ2Compressor diff_stream; - BZ2Compressor extra_stream; - - // Write 3 blocks of data to diff stream & extra stream respectively. - std::vector diff_data = {"HelloWorld", "BspatchPatchTest", - "BspatchDiffData"}; - std::vector extra_data = {"HelloWorld!", "BZ2PatchReaderSmoke", - "BspatchExtraData"}; - std::vector offset_increment = {100, 200, 300}; - for (size_t i = 0; i < diff_data.size(); i++) { - uint8_t buf[24]; - EncodeInt64(diff_data[i].size(), buf); - EncodeInt64(extra_data[i].size(), buf + 8); - EncodeInt64(offset_increment[i], buf + 16); - EXPECT_TRUE(ctrl_stream.Write(buf, sizeof(buf))); - EXPECT_TRUE( - diff_stream.Write(reinterpret_cast(diff_data[i].data()), - diff_data[i].size())); - EXPECT_TRUE(extra_stream.Write( - reinterpret_cast(extra_data[i].data()), - extra_data[i].size())); +class PatchReaderTest : public testing::Test { + protected: + void CompressData() { + for (size_t i = 0; i < diff_data_.size(); i++) { + uint8_t buf[24]; + EncodeInt64(diff_data_[i].size(), buf); + EncodeInt64(extra_data_[i].size(), buf + 8); + EncodeInt64(offset_increment_[i], buf + 16); + EXPECT_TRUE(ctrl_stream_->Write(buf, sizeof(buf))); + EXPECT_TRUE(diff_stream_->Write( + reinterpret_cast(diff_data_[i].data()), + diff_data_[i].size())); + EXPECT_TRUE(extra_stream_->Write( + reinterpret_cast(extra_data_[i].data()), + extra_data_[i].size())); + } + EXPECT_TRUE(ctrl_stream_->Finish()); + EXPECT_TRUE(diff_stream_->Finish()); + EXPECT_TRUE(extra_stream_->Finish()); } - EXPECT_TRUE(ctrl_stream.Finish()); - EXPECT_TRUE(diff_stream.Finish()); - EXPECT_TRUE(extra_stream.Finish()); - // Encode the header - uint8_t buf[24]; - EncodeInt64(ctrl_stream.GetCompressedData().size(), buf); - EncodeInt64(diff_stream.GetCompressedData().size(), buf + 8); - EncodeInt64(500, buf + 16); + void ConstructPatchData(std::vector* patch_data) { + EXPECT_EQ(static_cast(8), patch_data->size()); + // Encode the header + uint8_t buf[24]; + EncodeInt64(ctrl_stream_->GetCompressedData().size(), buf); + EncodeInt64(diff_stream_->GetCompressedData().size(), buf + 8); + EncodeInt64(new_file_size_, buf + 16); + std::copy(buf, buf + sizeof(buf), std::back_inserter(*patch_data)); + + // Concatenate the three streams into one patch. + std::copy(ctrl_stream_->GetCompressedData().begin(), + ctrl_stream_->GetCompressedData().end(), + std::back_inserter(*patch_data)); + std::copy(diff_stream_->GetCompressedData().begin(), + diff_stream_->GetCompressedData().end(), + std::back_inserter(*patch_data)); + std::copy(extra_stream_->GetCompressedData().begin(), + extra_stream_->GetCompressedData().end(), + std::back_inserter(*patch_data)); + } + + + void VerifyPatch(const std::vector& patch_data) { + BsdiffPatchReader patch_reader; + EXPECT_TRUE(patch_reader.Init(patch_data.data(), patch_data.size())); + EXPECT_EQ(new_file_size_, patch_reader.new_file_size()); + // Check that the decompressed data matches what we wrote. + for (size_t i = 0; i < diff_data_.size(); i++) { + ControlEntry control_entry(0, 0, 0); + EXPECT_TRUE(patch_reader.ParseControlEntry(&control_entry)); + EXPECT_EQ(diff_data_[i].size(), control_entry.diff_size); + EXPECT_EQ(extra_data_[i].size(), control_entry.extra_size); + EXPECT_EQ(offset_increment_[i], control_entry.offset_increment); + + uint8_t buffer[128] = {}; + EXPECT_TRUE(patch_reader.ReadDiffStream(buffer, diff_data_[i].size())); + EXPECT_EQ(0, memcmp(buffer, diff_data_[i].data(), diff_data_[i].size())); + EXPECT_TRUE(patch_reader.ReadExtraStream(buffer, extra_data_[i].size())); + EXPECT_EQ(0, + memcmp(buffer, extra_data_[i].data(), extra_data_[i].size())); + } + EXPECT_TRUE(patch_reader.Finish()); + } + + size_t new_file_size_{500}; + std::vector diff_data_{"HelloWorld", "BspatchPatchTest", + "BspatchDiffData"}; + std::vector extra_data_{"HelloWorld!", "BZ2PatchReaderSmoke", + "BspatchExtraData"}; + std::vector offset_increment_{100, 200, 300}; + + // The compressor streams. + std::unique_ptr ctrl_stream_{nullptr}; + std::unique_ptr diff_stream_{nullptr}; + std::unique_ptr extra_stream_{nullptr}; +}; + +TEST_F(PatchReaderTest, BZ2PatchReaderLegacyFormatSmoke) { + ctrl_stream_.reset(new BZ2Compressor()); + diff_stream_.reset(new BZ2Compressor()); + extra_stream_.reset(new BZ2Compressor()); + + CompressData(); std::vector patch_data; - std::copy(kMagicHeader, kMagicHeader + 8, std::back_inserter(patch_data)); - std::copy(buf, buf + sizeof(buf), std::back_inserter(patch_data)); + std::copy(kLegacyMagicHeader, kLegacyMagicHeader + 8, + std::back_inserter(patch_data)); + ConstructPatchData(&patch_data); - // Concatenate the three streams into one patch. - std::copy(ctrl_stream.GetCompressedData().begin(), - ctrl_stream.GetCompressedData().end(), - std::back_inserter(patch_data)); - std::copy(diff_stream.GetCompressedData().begin(), - diff_stream.GetCompressedData().end(), - std::back_inserter(patch_data)); - std::copy(extra_stream.GetCompressedData().begin(), - extra_stream.GetCompressedData().end(), - std::back_inserter(patch_data)); + VerifyPatch(patch_data); +} - BsdiffPatchReader patch_reader; - EXPECT_TRUE(patch_reader.Init(patch_data.data(), patch_data.size())); - EXPECT_EQ(500ull, patch_reader.new_file_size()); - // Check that the decompressed data matches what we wrote. - for (size_t i = 0; i < diff_data.size(); i++) { - ControlEntry control_entry(0, 0, 0); - EXPECT_TRUE(patch_reader.ParseControlEntry(&control_entry)); - uint8_t buffer[128] = {}; - EXPECT_TRUE(patch_reader.ReadDiffStream(buffer, diff_data[i].size())); - EXPECT_EQ(0, memcmp(buffer, diff_data[i].data(), diff_data[i].size())); - EXPECT_TRUE(patch_reader.ReadExtraStream(buffer, extra_data[i].size())); - EXPECT_EQ(0, memcmp(buffer, extra_data[i].data(), extra_data[i].size())); - } - EXPECT_TRUE(patch_reader.Finish()); +TEST_F(PatchReaderTest, BZ2PatchReaderNewFormatSmoke) { + // Compress the data with one bz2 and two brotli compressors. + ctrl_stream_.reset(new BZ2Compressor()); + diff_stream_.reset(new BrotliCompressor()); + extra_stream_.reset(new BrotliCompressor()); + + CompressData(); + + std::vector patch_data; + std::copy(kBSDF2MagicHeader, kBSDF2MagicHeader + 5, + std::back_inserter(patch_data)); + patch_data.push_back(static_cast(CompressorType::kBZ2)); + patch_data.push_back(static_cast(CompressorType::kBrotli)); + patch_data.push_back(static_cast(CompressorType::kBrotli)); + ConstructPatchData(&patch_data); + + VerifyPatch(patch_data); } } // namespace bsdiff diff --git a/patch_writer.cc b/patch_writer.cc index 8354825..f4f7930 100644 --- a/patch_writer.cc +++ b/patch_writer.cc @@ -29,14 +29,27 @@ void EncodeInt64(int64_t x, uint8_t* buf) { namespace bsdiff { BsdiffPatchWriter::BsdiffPatchWriter(const std::string& patch_filename, - CompressorType type) - : patch_filename_(patch_filename) { - ctrl_stream_ = CreateCompressor(type); - diff_stream_ = CreateCompressor(type); - extra_stream_ = CreateCompressor(type); + BsdiffFormat format) + : patch_filename_(patch_filename), format_(format) { + if (format_ == BsdiffFormat::kLegacy) { + ctrl_stream_ = CreateCompressor(CompressorType::kBZ2); + diff_stream_ = CreateCompressor(CompressorType::kBZ2); + extra_stream_ = CreateCompressor(CompressorType::kBZ2); + } else { + // TODO(xunchang) set different compression method according to the + // compressed size for these streams. + ctrl_stream_ = CreateCompressor(CompressorType::kBrotli); + diff_stream_ = CreateCompressor(CompressorType::kBrotli); + extra_stream_ = CreateCompressor(CompressorType::kBrotli); + } } bool BsdiffPatchWriter::Init(size_t /* new_size */) { + if (!(ctrl_stream_ && diff_stream_ && extra_stream_)) { + LOG(ERROR) << "uninitialized compressor stream" << endl; + return false; + } + fp_ = fopen(patch_filename_.c_str(), "w"); if (!fp_) { LOG(ERROR) << "Opening " << patch_filename_ << endl; @@ -107,19 +120,36 @@ bool BsdiffPatchWriter::Close() { bool BsdiffPatchWriter::WriteHeader(uint64_t ctrl_size, uint64_t diff_size) { /* Header format is - * 0 8 "BSDIFF40" - * 8 8 length of bzip2ed ctrl block - * 16 8 length of bzip2ed diff block + * 0 8 magic header + * 8 8 length of compressed ctrl block + * 16 8 length of compressed diff block * 24 8 length of new file * * File format is * 0 32 Header - * 32 ?? Bzip2ed ctrl block - * ?? ?? Bzip2ed diff block - * ?? ?? Bzip2ed extra block + * 32 ?? compressed ctrl block + * ?? ?? compressed diff block + * ?? ?? compressed extra block */ uint8_t header[32]; - memcpy(header, kMagicHeader, 8); + if (format_ == BsdiffFormat::kLegacy) { + // The magic header is "BSDIFF40" for legacy format. + memcpy(header, kLegacyMagicHeader, 8); + } else if (format_ == BsdiffFormat::kBsdf2) { + // The magic header for BSDF2 format: + // 0 5 BSDF2 + // 5 1 compressed type for control stream + // 6 1 compressed type for diff stream + // 7 1 compressed type for extra stream + memcpy(header, kBSDF2MagicHeader, 5); + header[5] = static_cast(ctrl_stream_->Type()); + header[6] = static_cast(diff_stream_->Type()); + header[7] = static_cast(extra_stream_->Type()); + } else { + LOG(ERROR) << "Unsupported bsdiff format." << endl; + return false; + } + EncodeInt64(ctrl_size, header + 8); EncodeInt64(diff_size, header + 16); EncodeInt64(written_output_, header + 24); diff --git a/patch_writer.h b/patch_writer.h index 03e7c83..ca713cb 100644 --- a/patch_writer.h +++ b/patch_writer.h @@ -20,7 +20,7 @@ class BsdiffPatchWriter : public PatchWriterInterface { public: // Create the patch writer using |type| as the compression algorithm and the // file |patch_filename| to write the patch data. - BsdiffPatchWriter(const std::string& patch_filename, CompressorType type); + BsdiffPatchWriter(const std::string& patch_filename, BsdiffFormat format); // PatchWriterInterface overrides. bool Init(size_t new_size) override; @@ -42,6 +42,9 @@ class BsdiffPatchWriter : public PatchWriterInterface { FILE* fp_{nullptr}; std::string patch_filename_; + // The format of bsdiff we're using. + BsdiffFormat format_; + // The three internal compressed streams. std::unique_ptr ctrl_stream_{nullptr}; std::unique_ptr diff_stream_{nullptr}; diff --git a/patch_writer_factory.cc b/patch_writer_factory.cc index d86bafc..d24cffe 100644 --- a/patch_writer_factory.cc +++ b/patch_writer_factory.cc @@ -9,11 +9,11 @@ namespace bsdiff { -// TODO(xunchang) choose the compression type based on the input parameter. +// TODO(xunchang) choose the bsdiff format based on the input parameter. std::unique_ptr CreateBsdiffPatchWriter( const std::string& patch_filename) { return std::unique_ptr( - new BsdiffPatchWriter(patch_filename, CompressorType::kBZ2)); + new BsdiffPatchWriter(patch_filename, BsdiffFormat::kLegacy)); } std::unique_ptr CreateEndsleyPatchWriter( diff --git a/patch_writer_unittest.cc b/patch_writer_unittest.cc index c83d0e7..d429a0a 100644 --- a/patch_writer_unittest.cc +++ b/patch_writer_unittest.cc @@ -53,7 +53,8 @@ class BsdiffPatchWriterTest : public testing::Test { } test_utils::ScopedTempFile patch_file_{"bsdiff_newfile.XXXXXX"}; - BsdiffPatchWriter patch_writer_{patch_file_.filename(), CompressorType::kBZ2}; + BsdiffPatchWriter patch_writer_{patch_file_.filename(), + BsdiffFormat::kLegacy}; }; TEST_F(BsdiffPatchWriterTest, CreateEmptyPatchTest) {