Merge "Implement a new bsdiff format" am: 2ab0083e1e

am: 1ab7aa7a4a

Change-Id: Ieea3178d573e5901b4aaa38c2178a2561c8fc458
This commit is contained in:
Tianjie Xu authored and android-build-merger committed 2017-11-17 19:09:20 +00:00
commit 1046772273
15 files changed
+247 -105

No files matched your search

+1 -1
View File
@@ -12,7 +12,7 @@ namespace {
// TODO(xunchang) set brotli compression parameters based on input options.
const size_t kBufferSize = 1024 * 1024;
const uint32_t kBrotliQuality = 11;
const uint32_t kBrotliLgwin = 24;
const uint32_t kBrotliLgwin = 20;
} // namespace
+2
View File
@@ -13,6 +13,7 @@
#include "bsdiff/compressor_buffer.h"
#include "bsdiff/compressor_interface.h"
#include "bsdiff/constants.h"
namespace bsdiff {
@@ -25,6 +26,7 @@ class BrotliCompressor : public CompressorInterface {
bool Write(const uint8_t* buf, size_t size) override;
bool Finish() override;
const std::vector<uint8_t>& GetCompressedData() override;
CompressorType Type() override { return CompressorType::kBrotli; }
private:
BrotliEncoderState* brotli_encoder_state_;
+1 -1
View File
@@ -49,7 +49,7 @@ namespace bsdiff {
int bsdiff(const uint8_t* old_buf, size_t oldsize, const uint8_t* new_buf,
size_t newsize, const char* patch_filename,
SuffixArrayIndexInterface** sai_cache) {
BsdiffPatchWriter patch(patch_filename, CompressorType::kBZ2);
BsdiffPatchWriter patch(patch_filename, BsdiffFormat::kLegacy);
return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, sai_cache);
}
+1
View File
@@ -29,6 +29,7 @@ class BZ2Compressor : public CompressorInterface {
bool Write(const uint8_t* buf, size_t size) override;
bool Finish() override;
const std::vector<uint8_t>& GetCompressedData() override;
CompressorType Type() override { return CompressorType::kBZ2; }
private:
// The low-level bzip2 stream.
+13 -7
View File
@@ -2,19 +2,25 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <bsdiff/compressor_interface.h>
#include "bsdiff/compressor_interface.h"
#include <bsdiff/brotli_compressor.h>
#include <bsdiff/bz2_compressor.h>
#include "bsdiff/brotli_compressor.h"
#include "bsdiff/bz2_compressor.h"
#include "bsdiff/logging.h"
namespace bsdiff {
std::unique_ptr<CompressorInterface> CreateCompressor(CompressorType type) {
if (type == CompressorType::kBrotli) {
return std::unique_ptr<CompressorInterface>(new BrotliCompressor());
switch (type) {
case CompressorType::kBZ2:
return std::unique_ptr<CompressorInterface>(new BZ2Compressor());
case CompressorType::kBrotli:
return std::unique_ptr<CompressorInterface>(new BrotliCompressor());
default:
LOG(ERROR) << "unsupported compressor type: "
<< static_cast<uint8_t>(type) << std::endl;
return nullptr;
}
// Use BZ2 as a default compressor.
return std::unique_ptr<CompressorInterface>(new BZ2Compressor());
}
} // namespace bsdiff
+3
View File
@@ -28,6 +28,9 @@ class CompressorInterface {
// Return the compressed data. This method must be only called after Finish().
virtual const std::vector<uint8_t>& GetCompressedData() = 0;
// Return the type of the current compressor.
virtual CompressorType Type() = 0;
protected:
CompressorInterface() = default;
};
+9 -5
View File
@@ -9,15 +9,19 @@
namespace bsdiff {
enum class CompressorType {
kBrotli,
kBZ2,
enum class CompressorType : uint8_t {
kNoCompression = 0,
kBZ2 = 1,
kBrotli = 2,
};
// The header of the upstream's "BSDIFF40" format using BZ2 as compressor.
constexpr uint8_t kMagicHeader[] = "BSDIFF40";
constexpr uint8_t kLegacyMagicHeader[] = "BSDIFF40";
// The header of the new BSDF2 format. This format supports both bz2 and
// brotli compressor.
constexpr uint8_t kBSDF2MagicHeader[] = "BSDF2";
} // namespace bsdiff
#endif // _BSDIFF_CONSTANTS_H_
+10 -4
View File
@@ -6,15 +6,21 @@
#include "bsdiff/brotli_decompressor.h"
#include "bsdiff/bz2_decompressor.h"
#include "bsdiff/logging.h"
namespace bsdiff {
std::unique_ptr<DecompressorInterface> CreateDecompressor(CompressorType type) {
if (type == CompressorType::kBrotli) {
return std::unique_ptr<DecompressorInterface>(new BrotliDecompressor());
switch (type) {
case CompressorType::kBZ2:
return std::unique_ptr<DecompressorInterface>(new BZ2Decompressor());
case CompressorType::kBrotli:
return std::unique_ptr<DecompressorInterface>(new BrotliDecompressor());
default:
LOG(ERROR) << "unsupported compressor type: "
<< static_cast<uint8_t>(type) << std::endl;
return nullptr;
}
// Use BZ2 as a default decompressor.
return std::unique_ptr<DecompressorInterface>(new BZ2Decompressor());
}
} // namespace bsdiff
+5
View File
@@ -12,6 +12,11 @@
namespace bsdiff {
enum class BsdiffFormat {
kLegacy,
kBsdf2,
};
class PatchWriterInterface {
public:
virtual ~PatchWriterInterface() = default;
+45 -13
View File
@@ -20,8 +20,8 @@ using std::endl;
namespace bsdiff {
bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) {
// File format:
// 0 8 "BSDIFF40"
// File format:
// 0 8 magic header
// 8 8 X
// 16 8 Y
// 24 8 new_file_size
@@ -32,8 +32,38 @@ bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) {
// from oldfile to x bytes from the diff block; copy y bytes from the
// extra block; seek forwards in oldfile by z bytes".
if (patch_size < 32) {
LOG(ERROR) << "Too small to be a bspatch." << endl;
return false;
}
// Check for appropriate magic.
if (memcmp(patch_data, kMagicHeader, 8) != 0) {
std::vector<CompressorType> compression_type;
if (memcmp(patch_data, kLegacyMagicHeader, 8) == 0) {
// The magic header is "BSDIFF40" for legacy format.
compression_type = {CompressorType::kBZ2, CompressorType::kBZ2,
CompressorType::kBZ2};
} else if (memcmp(patch_data, kBSDF2MagicHeader, 5) == 0) {
// The magic header for BSDF2 format:
// 0 5 BSDF2
// 5 1 compressed type for control stream
// 6 1 compressed type for diff stream
// 7 1 compressed type for extra stream
for (size_t i = 5; i < 8; i++) {
uint8_t type = patch_data[i];
switch (type) {
case static_cast<uint8_t>(CompressorType::kBZ2):
compression_type.push_back(CompressorType::kBZ2);
break;
case static_cast<uint8_t>(CompressorType::kBrotli):
compression_type.push_back(CompressorType::kBrotli);
break;
default:
LOG(ERROR) << "Unsupported compression type: "
<< static_cast<int>(type) << endl;
return false;
}
}
} else {
LOG(ERROR) << "Not a bsdiff patch." << endl;
return false;
}
@@ -47,18 +77,20 @@ bool BsdiffPatchReader::Init(const uint8_t* patch_data, size_t patch_size) {
LOG(ERROR) << "Corrupt patch. ctrl_len: " << ctrl_len
<< ", data_len: " << diff_len
<< ", new_file_size: " << signed_newsize
<< ", patch_size: " << patch_size;
<< ", patch_size: " << patch_size << endl;
return false;
}
new_file_size_ = signed_newsize;
// TODO(xunchang) set the correct decompressor based on the info in the
// header.
ctrl_stream_ = CreateDecompressor(CompressorType::kBZ2);
diff_stream_ = CreateDecompressor(CompressorType::kBZ2);
extra_stream_ = CreateDecompressor(CompressorType::kBZ2);
ctrl_stream_ = CreateDecompressor(compression_type[0]);
diff_stream_ = CreateDecompressor(compression_type[1]);
extra_stream_ = CreateDecompressor(compression_type[2]);
if (!(ctrl_stream_ && diff_stream_ && extra_stream_)) {
LOG(ERROR) << "uninitialized decompressor stream" << endl;
return false;
}
int64_t offset = 32;
size_t offset = 32;
if (!ctrl_stream_->SetInputData(const_cast<uint8_t*>(patch_data) + offset,
ctrl_len)) {
LOG(ERROR) << "Failed to init ctrl stream, ctrl_len: " << ctrl_len << endl;
@@ -122,17 +154,17 @@ bool BsdiffPatchReader::ReadExtraStream(uint8_t* buf, size_t size) {
bool BsdiffPatchReader::Finish() {
if (!ctrl_stream_->Close()) {
LOG(ERROR) << "Failed to close the control stream";
LOG(ERROR) << "Failed to close the control stream" << endl;
return false;
}
if (!diff_stream_->Close()) {
LOG(ERROR) << "Failed to close the diff stream";
LOG(ERROR) << "Failed to close the diff stream" << endl;
return false;
}
if (!extra_stream_->Close()) {
LOG(ERROR) << "Failed to close the extra stream";
LOG(ERROR) << "Failed to close the extra stream" << endl;
return false;
}
return true;
+107 -58
View File
@@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "bsdiff/patch_reader.h"
#include <unistd.h>
#include <algorithm>
@@ -10,8 +12,8 @@
#include <gtest/gtest.h>
#include "bsdiff/brotli_compressor.h"
#include "bsdiff/bz2_compressor.h"
#include "bsdiff/patch_reader.h"
#include "bsdiff/utils.h"
namespace {
@@ -28,69 +30,116 @@ void EncodeInt64(int64_t x, uint8_t* buf) {
namespace bsdiff {
TEST(PatchReaderTest, BZ2PatchReaderSmoke) {
BZ2Compressor ctrl_stream;
BZ2Compressor diff_stream;
BZ2Compressor extra_stream;
// Write 3 blocks of data to diff stream & extra stream respectively.
std::vector<std::string> diff_data = {"HelloWorld", "BspatchPatchTest",
"BspatchDiffData"};
std::vector<std::string> extra_data = {"HelloWorld!", "BZ2PatchReaderSmoke",
"BspatchExtraData"};
std::vector<int64_t> offset_increment = {100, 200, 300};
for (size_t i = 0; i < diff_data.size(); i++) {
uint8_t buf[24];
EncodeInt64(diff_data[i].size(), buf);
EncodeInt64(extra_data[i].size(), buf + 8);
EncodeInt64(offset_increment[i], buf + 16);
EXPECT_TRUE(ctrl_stream.Write(buf, sizeof(buf)));
EXPECT_TRUE(
diff_stream.Write(reinterpret_cast<const uint8_t*>(diff_data[i].data()),
diff_data[i].size()));
EXPECT_TRUE(extra_stream.Write(
reinterpret_cast<const uint8_t*>(extra_data[i].data()),
extra_data[i].size()));
class PatchReaderTest : public testing::Test {
protected:
void CompressData() {
for (size_t i = 0; i < diff_data_.size(); i++) {
uint8_t buf[24];
EncodeInt64(diff_data_[i].size(), buf);
EncodeInt64(extra_data_[i].size(), buf + 8);
EncodeInt64(offset_increment_[i], buf + 16);
EXPECT_TRUE(ctrl_stream_->Write(buf, sizeof(buf)));
EXPECT_TRUE(diff_stream_->Write(
reinterpret_cast<const uint8_t*>(diff_data_[i].data()),
diff_data_[i].size()));
EXPECT_TRUE(extra_stream_->Write(
reinterpret_cast<const uint8_t*>(extra_data_[i].data()),
extra_data_[i].size()));
}
EXPECT_TRUE(ctrl_stream_->Finish());
EXPECT_TRUE(diff_stream_->Finish());
EXPECT_TRUE(extra_stream_->Finish());
}
EXPECT_TRUE(ctrl_stream.Finish());
EXPECT_TRUE(diff_stream.Finish());
EXPECT_TRUE(extra_stream.Finish());
// Encode the header
uint8_t buf[24];
EncodeInt64(ctrl_stream.GetCompressedData().size(), buf);
EncodeInt64(diff_stream.GetCompressedData().size(), buf + 8);
EncodeInt64(500, buf + 16);
void ConstructPatchData(std::vector<uint8_t>* patch_data) {
EXPECT_EQ(static_cast<size_t>(8), patch_data->size());
// Encode the header
uint8_t buf[24];
EncodeInt64(ctrl_stream_->GetCompressedData().size(), buf);
EncodeInt64(diff_stream_->GetCompressedData().size(), buf + 8);
EncodeInt64(new_file_size_, buf + 16);
std::copy(buf, buf + sizeof(buf), std::back_inserter(*patch_data));
// Concatenate the three streams into one patch.
std::copy(ctrl_stream_->GetCompressedData().begin(),
ctrl_stream_->GetCompressedData().end(),
std::back_inserter(*patch_data));
std::copy(diff_stream_->GetCompressedData().begin(),
diff_stream_->GetCompressedData().end(),
std::back_inserter(*patch_data));
std::copy(extra_stream_->GetCompressedData().begin(),
extra_stream_->GetCompressedData().end(),
std::back_inserter(*patch_data));
}
void VerifyPatch(const std::vector<uint8_t>& patch_data) {
BsdiffPatchReader patch_reader;
EXPECT_TRUE(patch_reader.Init(patch_data.data(), patch_data.size()));
EXPECT_EQ(new_file_size_, patch_reader.new_file_size());
// Check that the decompressed data matches what we wrote.
for (size_t i = 0; i < diff_data_.size(); i++) {
ControlEntry control_entry(0, 0, 0);
EXPECT_TRUE(patch_reader.ParseControlEntry(&control_entry));
EXPECT_EQ(diff_data_[i].size(), control_entry.diff_size);
EXPECT_EQ(extra_data_[i].size(), control_entry.extra_size);
EXPECT_EQ(offset_increment_[i], control_entry.offset_increment);
uint8_t buffer[128] = {};
EXPECT_TRUE(patch_reader.ReadDiffStream(buffer, diff_data_[i].size()));
EXPECT_EQ(0, memcmp(buffer, diff_data_[i].data(), diff_data_[i].size()));
EXPECT_TRUE(patch_reader.ReadExtraStream(buffer, extra_data_[i].size()));
EXPECT_EQ(0,
memcmp(buffer, extra_data_[i].data(), extra_data_[i].size()));
}
EXPECT_TRUE(patch_reader.Finish());
}
size_t new_file_size_{500};
std::vector<std::string> diff_data_{"HelloWorld", "BspatchPatchTest",
"BspatchDiffData"};
std::vector<std::string> extra_data_{"HelloWorld!", "BZ2PatchReaderSmoke",
"BspatchExtraData"};
std::vector<int64_t> offset_increment_{100, 200, 300};
// The compressor streams.
std::unique_ptr<CompressorInterface> ctrl_stream_{nullptr};
std::unique_ptr<CompressorInterface> diff_stream_{nullptr};
std::unique_ptr<CompressorInterface> extra_stream_{nullptr};
};
TEST_F(PatchReaderTest, BZ2PatchReaderLegacyFormatSmoke) {
ctrl_stream_.reset(new BZ2Compressor());
diff_stream_.reset(new BZ2Compressor());
extra_stream_.reset(new BZ2Compressor());
CompressData();
std::vector<uint8_t> patch_data;
std::copy(kMagicHeader, kMagicHeader + 8, std::back_inserter(patch_data));
std::copy(buf, buf + sizeof(buf), std::back_inserter(patch_data));
std::copy(kLegacyMagicHeader, kLegacyMagicHeader + 8,
std::back_inserter(patch_data));
ConstructPatchData(&patch_data);
// Concatenate the three streams into one patch.
std::copy(ctrl_stream.GetCompressedData().begin(),
ctrl_stream.GetCompressedData().end(),
std::back_inserter(patch_data));
std::copy(diff_stream.GetCompressedData().begin(),
diff_stream.GetCompressedData().end(),
std::back_inserter(patch_data));
std::copy(extra_stream.GetCompressedData().begin(),
extra_stream.GetCompressedData().end(),
std::back_inserter(patch_data));
VerifyPatch(patch_data);
}
BsdiffPatchReader patch_reader;
EXPECT_TRUE(patch_reader.Init(patch_data.data(), patch_data.size()));
EXPECT_EQ(500ull, patch_reader.new_file_size());
// Check that the decompressed data matches what we wrote.
for (size_t i = 0; i < diff_data.size(); i++) {
ControlEntry control_entry(0, 0, 0);
EXPECT_TRUE(patch_reader.ParseControlEntry(&control_entry));
uint8_t buffer[128] = {};
EXPECT_TRUE(patch_reader.ReadDiffStream(buffer, diff_data[i].size()));
EXPECT_EQ(0, memcmp(buffer, diff_data[i].data(), diff_data[i].size()));
EXPECT_TRUE(patch_reader.ReadExtraStream(buffer, extra_data[i].size()));
EXPECT_EQ(0, memcmp(buffer, extra_data[i].data(), extra_data[i].size()));
}
EXPECT_TRUE(patch_reader.Finish());
TEST_F(PatchReaderTest, BZ2PatchReaderNewFormatSmoke) {
// Compress the data with one bz2 and two brotli compressors.
ctrl_stream_.reset(new BZ2Compressor());
diff_stream_.reset(new BrotliCompressor());
extra_stream_.reset(new BrotliCompressor());
CompressData();
std::vector<uint8_t> patch_data;
std::copy(kBSDF2MagicHeader, kBSDF2MagicHeader + 5,
std::back_inserter(patch_data));
patch_data.push_back(static_cast<uint8_t>(CompressorType::kBZ2));
patch_data.push_back(static_cast<uint8_t>(CompressorType::kBrotli));
patch_data.push_back(static_cast<uint8_t>(CompressorType::kBrotli));
ConstructPatchData(&patch_data);
VerifyPatch(patch_data);
}
} // namespace bsdiff
+42 -12
View File
@@ -29,14 +29,27 @@ void EncodeInt64(int64_t x, uint8_t* buf) {
namespace bsdiff {
BsdiffPatchWriter::BsdiffPatchWriter(const std::string& patch_filename,
CompressorType type)
: patch_filename_(patch_filename) {
ctrl_stream_ = CreateCompressor(type);
diff_stream_ = CreateCompressor(type);
extra_stream_ = CreateCompressor(type);
BsdiffFormat format)
: patch_filename_(patch_filename), format_(format) {
if (format_ == BsdiffFormat::kLegacy) {
ctrl_stream_ = CreateCompressor(CompressorType::kBZ2);
diff_stream_ = CreateCompressor(CompressorType::kBZ2);
extra_stream_ = CreateCompressor(CompressorType::kBZ2);
} else {
// TODO(xunchang) set different compression method according to the
// compressed size for these streams.
ctrl_stream_ = CreateCompressor(CompressorType::kBrotli);
diff_stream_ = CreateCompressor(CompressorType::kBrotli);
extra_stream_ = CreateCompressor(CompressorType::kBrotli);
}
}
bool BsdiffPatchWriter::Init(size_t /* new_size */) {
if (!(ctrl_stream_ && diff_stream_ && extra_stream_)) {
LOG(ERROR) << "uninitialized compressor stream" << endl;
return false;
}
fp_ = fopen(patch_filename_.c_str(), "w");
if (!fp_) {
LOG(ERROR) << "Opening " << patch_filename_ << endl;
@@ -107,19 +120,36 @@ bool BsdiffPatchWriter::Close() {
bool BsdiffPatchWriter::WriteHeader(uint64_t ctrl_size, uint64_t diff_size) {
/* Header format is
* 0 8 "BSDIFF40"
* 8 8 length of bzip2ed ctrl block
* 16 8 length of bzip2ed diff block
* 0 8 magic header
* 8 8 length of compressed ctrl block
* 16 8 length of compressed diff block
* 24 8 length of new file
*
* File format is
* 0 32 Header
* 32 ?? Bzip2ed ctrl block
* ?? ?? Bzip2ed diff block
* ?? ?? Bzip2ed extra block
* 32 ?? compressed ctrl block
* ?? ?? compressed diff block
* ?? ?? compressed extra block
*/
uint8_t header[32];
memcpy(header, kMagicHeader, 8);
if (format_ == BsdiffFormat::kLegacy) {
// The magic header is "BSDIFF40" for legacy format.
memcpy(header, kLegacyMagicHeader, 8);
} else if (format_ == BsdiffFormat::kBsdf2) {
// The magic header for BSDF2 format:
// 0 5 BSDF2
// 5 1 compressed type for control stream
// 6 1 compressed type for diff stream
// 7 1 compressed type for extra stream
memcpy(header, kBSDF2MagicHeader, 5);
header[5] = static_cast<uint8_t>(ctrl_stream_->Type());
header[6] = static_cast<uint8_t>(diff_stream_->Type());
header[7] = static_cast<uint8_t>(extra_stream_->Type());
} else {
LOG(ERROR) << "Unsupported bsdiff format." << endl;
return false;
}
EncodeInt64(ctrl_size, header + 8);
EncodeInt64(diff_size, header + 16);
EncodeInt64(written_output_, header + 24);
+4 -1
View File
@@ -20,7 +20,7 @@ class BsdiffPatchWriter : public PatchWriterInterface {
public:
// Create the patch writer using |type| as the compression algorithm and the
// file |patch_filename| to write the patch data.
BsdiffPatchWriter(const std::string& patch_filename, CompressorType type);
BsdiffPatchWriter(const std::string& patch_filename, BsdiffFormat format);
// PatchWriterInterface overrides.
bool Init(size_t new_size) override;
@@ -42,6 +42,9 @@ class BsdiffPatchWriter : public PatchWriterInterface {
FILE* fp_{nullptr};
std::string patch_filename_;
// The format of bsdiff we're using.
BsdiffFormat format_;
// The three internal compressed streams.
std::unique_ptr<CompressorInterface> ctrl_stream_{nullptr};
std::unique_ptr<CompressorInterface> diff_stream_{nullptr};
+2 -2
View File
@@ -9,11 +9,11 @@
namespace bsdiff {
// TODO(xunchang) choose the compression type based on the input parameter.
// TODO(xunchang) choose the bsdiff format based on the input parameter.
std::unique_ptr<PatchWriterInterface> CreateBsdiffPatchWriter(
const std::string& patch_filename) {
return std::unique_ptr<PatchWriterInterface>(
new BsdiffPatchWriter(patch_filename, CompressorType::kBZ2));
new BsdiffPatchWriter(patch_filename, BsdiffFormat::kLegacy));
}
std::unique_ptr<PatchWriterInterface> CreateEndsleyPatchWriter(
+2 -1
View File
@@ -53,7 +53,8 @@ class BsdiffPatchWriterTest : public testing::Test {
}
test_utils::ScopedTempFile patch_file_{"bsdiff_newfile.XXXXXX"};
BsdiffPatchWriter patch_writer_{patch_file_.filename(), CompressorType::kBZ2};
BsdiffPatchWriter patch_writer_{patch_file_.filename(),
BsdiffFormat::kLegacy};
};
TEST_F(BsdiffPatchWriterTest, CreateEmptyPatchTest) {