diff --git a/Android.bp b/Android.bp index 4566caa..fc224e9 100644 --- a/Android.bp +++ b/Android.bp @@ -59,6 +59,7 @@ cc_library_static { "compressor_buffer.cc", "compressor_interface.cc", "diff_encoder.cc", + "endsley_patch_writer.cc", "patch_writer.cc", "patch_writer_factory.cc", "split_patch_writer.cc", @@ -109,6 +110,7 @@ cc_test { "bsdiff_unittest.cc", "bspatch_unittest.cc", "diff_encoder_unittest.cc", + "endsley_patch_writer_unittest.cc", "extents_file_unittest.cc", "extents_unittest.cc", "patch_reader_unittest.cc", diff --git a/Makefile b/Makefile index b76e3cb..6a06440 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ bsdiff_src_files := \ compressor_buffer.cc \ compressor_interface.cc \ diff_encoder.cc \ + endsley_patch_writer.cc \ patch_writer.cc \ patch_writer_factory.cc \ split_patch_writer.cc \ @@ -63,6 +64,7 @@ bsdiff_common_unittests := \ bsdiff_unittest.cc \ bspatch_unittest.cc \ diff_encoder_unittest.cc \ + endsley_patch_writer_unittest.cc \ extents_file_unittest.cc \ extents_unittest.cc \ patch_reader_unittest.cc \ diff --git a/bsdiff.gyp b/bsdiff.gyp index 3be7108..14d8c9a 100644 --- a/bsdiff.gyp +++ b/bsdiff.gyp @@ -62,6 +62,7 @@ 'compressor_buffer.cc', 'compressor_interface.cc', 'diff_encoder.cc', + 'endsley_patch_writer.cc', 'patch_writer.cc', 'patch_writer_factory.cc', 'split_patch_writer.cc', @@ -140,6 +141,7 @@ 'bsdiff_unittest.cc', 'bspatch_unittest.cc', 'diff_encoder_unittest.cc', + 'endsley_patch_writer_unittest.cc', 'extents_file_unittest.cc', 'extents_unittest.cc', 'patch_reader_unittest.cc', diff --git a/endsley_patch_writer.cc b/endsley_patch_writer.cc new file mode 100644 index 0000000..4e4a3f1 --- /dev/null +++ b/endsley_patch_writer.cc @@ -0,0 +1,178 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/endsley_patch_writer.h" + +#include + +#include + +#include "bsdiff/logging.h" + +using std::endl; + +namespace { + +constexpr uint8_t kEndsleyMagicHeader[] = "ENDSLEY/BSDIFF43"; + +void EncodeInt64(int64_t x, uint8_t* buf) { + uint64_t y = x < 0 ? (1ULL << 63ULL) - x : x; + for (int i = 0; i < 8; ++i) { + buf[i] = y & 0xff; + y /= 256; + } +} + +// The minimum size that we would consider flushing out. +constexpr size_t kMinimumFlushSize = 1024 * 1024; // 1 MiB + +} // namespace + +namespace bsdiff { + +bool EndsleyPatchWriter::Init(size_t new_size) { + // The patch is uncompressed and it will need exactly: + // new_size + 24 * len(control_entries) + sizeof(header) + // We don't know the length of the control entries yet, but we can reserve + // enough space to hold at least |new_size|. + patch_->clear(); + patch_->reserve(new_size); + + // Header is the magic followed by the new length. + uint8_t header[24]; + memcpy(header, kEndsleyMagicHeader, 16); + EncodeInt64(new_size, header + 16); + EmitBuffer(header, sizeof(header)); + return true; +} + +bool EndsleyPatchWriter::WriteDiffStream(const uint8_t* data, size_t size) { + if (!size) + return true; + // Speed-up the common case where the diff stream data is added right after + // the control entry that refers to it. + if (control_.empty() && pending_diff_ >= size) { + pending_diff_ -= size; + EmitBuffer(data, size); + return true; + } + + diff_data_.insert(diff_data_.end(), data, data + size); + return true; +} + +bool EndsleyPatchWriter::WriteExtraStream(const uint8_t* data, size_t size) { + if (!size) + return true; + // Speed-up the common case where the extra stream data is added right after + // the diff stream data and the control entry that refers to it. Note that + // the diff data comes first so we need to make sure it is all out. + if (control_.empty() && !pending_diff_ && pending_extra_ >= size) { + pending_extra_ -= size; + EmitBuffer(data, size); + return true; + } + + extra_data_.insert(extra_data_.end(), data, data + size); + return true; +} + +bool EndsleyPatchWriter::AddControlEntry(const ControlEntry& entry) { + // Speed-up the common case where the control entry is added when there's + // nothing else pending. + if (control_.empty() && diff_data_.empty() && extra_data_.empty() && + !pending_diff_ && !pending_extra_) { + pending_diff_ = entry.diff_size; + pending_extra_ = entry.extra_size; + EmitControlEntry(entry); + return true; + } + + control_.push_back(entry); + pending_control_data_ += entry.diff_size + entry.extra_size; + + // Check whether it is worth Flushing the entries now that the we have more + // control entries. We need control entries to write enough output data, and + // we need that output data to be at least 50% of the available diff and extra + // data. This last requirement is to reduce the overhead of removing the + // flushed data. + if (pending_control_data_ > kMinimumFlushSize && + (diff_data_.size() + extra_data_.size()) / 2 <= pending_control_data_) { + Flush(); + } + + return true; +} + +bool EndsleyPatchWriter::Close() { + // Flush any pending data. + Flush(); + + if (pending_diff_ || pending_extra_ || !control_.empty()) { + LOG(ERROR) << "Insufficient data sent to diff/extra streams" << endl; + return false; + } + + if (!diff_data_.empty() || !extra_data_.empty()) { + LOG(ERROR) << "Pending data to diff/extra not flushed out on Close()" + << endl; + return false; + } + return true; +} + +void EndsleyPatchWriter::EmitControlEntry(const ControlEntry& entry) { + // Generate the 24 byte control entry. + uint8_t buf[24]; + EncodeInt64(entry.diff_size, buf); + EncodeInt64(entry.extra_size, buf + 8); + EncodeInt64(entry.offset_increment, buf + 16); + EmitBuffer(buf, sizeof(buf)); +} + +void EndsleyPatchWriter::EmitBuffer(const uint8_t* data, size_t size) { + patch_->insert(patch_->end(), data, data + size); +} + +void EndsleyPatchWriter::Flush() { + size_t used_diff = 0; + size_t used_extra = 0; + size_t used_control = 0; + do { + if (!pending_diff_ && !pending_extra_ && used_control < control_.size()) { + // We can emit a control entry in these conditions. + const ControlEntry& entry = control_[used_control]; + used_control++; + + pending_diff_ = entry.diff_size; + pending_extra_ = entry.extra_size; + pending_control_data_ -= entry.extra_size + entry.diff_size; + EmitControlEntry(entry); + } + + if (pending_diff_) { + size_t diff_size = std::min(diff_data_.size() - used_diff, pending_diff_); + EmitBuffer(diff_data_.data() + used_diff, diff_size); + pending_diff_ -= diff_size; + used_diff += diff_size; + } + + if (!pending_diff_ && pending_extra_) { + size_t extra_size = + std::min(extra_data_.size() - used_extra, pending_extra_); + EmitBuffer(extra_data_.data() + used_extra, extra_size); + pending_extra_ -= extra_size; + used_extra += extra_size; + } + } while (!pending_diff_ && !pending_extra_ && used_control < control_.size()); + + if (used_diff) + diff_data_.erase(diff_data_.begin(), diff_data_.begin() + used_diff); + if (used_extra) + extra_data_.erase(extra_data_.begin(), extra_data_.begin() + used_extra); + if (used_control) + control_.erase(control_.begin(), control_.begin() + used_control); +} + +} // namespace bsdiff diff --git a/endsley_patch_writer.h b/endsley_patch_writer.h new file mode 100644 index 0000000..1840d8d --- /dev/null +++ b/endsley_patch_writer.h @@ -0,0 +1,83 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef _BSDIFF_ENDSLEY_PATCH_WRITER_H_ +#define _BSDIFF_ENDSLEY_PATCH_WRITER_H_ + +#include +#include + +#include "bsdiff/patch_writer_interface.h" + +namespace bsdiff { + +// A PatchWriterInterface class compatible with the format used by Android Play +// Store's bsdiff implementation, which is based on Matthew Endsley's bsdiff +// implementation. See https://github.com/mendsley/bsdiff for the original +// implementation of this format. See also Google's APK patch size estimator for +// more information on the file-by-file format used by Play Store: +// https://github.com/googlesamples/apk-patch-size-estimator + +// This format, identified by the "ENDSLEY/BSDIFF43" magic string, uses a single +// stream with the control entries, diff data and extra data interleaved. After +// the header, each Control Entry is stored in 24 bytes followed by the diff +// stream data for that entry only, and then followed by the extra stream data +// for that entry only. The format doesn't handle the compression of the data, +// instead, the whole file (including the magic string) is compressed with any +// compression algorithm. + +// This format is easier to parse and allows the patch to be streamed, but by +// mixing the diff and extra data into the same compression context offers a +// slightly worse compression ratio (about 3.5% compared to upstream's format). + +class EndsleyPatchWriter : public PatchWriterInterface { + public: + // Create the patch writer that will write the data to the passed vector + // |patch|, resizing it as needed. The |patch| vector must be valid until + // Close() is called or this patch is destroyed. + explicit EndsleyPatchWriter(std::vector* patch) : patch_(patch) {} + + // PatchWriterInterface overrides. + bool Init(size_t new_size) override; + bool WriteDiffStream(const uint8_t* data, size_t size) override; + bool WriteExtraStream(const uint8_t* data, size_t size) override; + bool AddControlEntry(const ControlEntry& entry) override; + bool Close() override; + + private: + // Emit at the end of the |patch_| vector the passed control entry. + void EmitControlEntry(const ControlEntry& entry); + + // Emit at the end of the |patch_| vector the passed buffer. + void EmitBuffer(const uint8_t* data, size_t size); + + // Flush as much as possible of the pending data. + void Flush(); + + // The vector we are writing to, owned by the caller. + std::vector* patch_; + + // The pending diff and extra data to be encoded in the file. These vectors + // would not be used whenever is possible to the data directly to the patch_ + // vector; namely when the control, diff and extra stream data are provided in + // that order for each control entry. + std::vector diff_data_; + std::vector extra_data_; + std::vector control_; + + // Defined as the sum of all the diff_size and extra_size values in + // |control_|. This is used to determine whether it is worth Flushing the + // pending data. + size_t pending_control_data_{0}; + + // Number of bytes in the diff and extra stream that are pending in the + // last control entry encoded in the |patch_|. If both are zero the last + // control entry was completely emitted. + size_t pending_diff_{0}; + size_t pending_extra_{0}; +}; + +} // namespace bsdiff + +#endif // _BSDIFF_ENDSLEY_PATCH_WRITER_H_ diff --git a/endsley_patch_writer_unittest.cc b/endsley_patch_writer_unittest.cc new file mode 100644 index 0000000..29f404c --- /dev/null +++ b/endsley_patch_writer_unittest.cc @@ -0,0 +1,158 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/endsley_patch_writer.h" + +#include + +#include + +namespace { + +std::vector VectorFromString(const std::string& s) { + return std::vector(s.data(), s.data() + s.size()); +} + +} // namespace + +namespace bsdiff { + +class EndsleyPatchWriterTest : public testing::Test { + protected: + // Return a subvector from |data_| starting at |start| of size at most |size|. + std::vector DataSubvector(size_t start, size_t size) { + if (start > data_.size()) + return std::vector(); + + size = std::min(size, data_.size() - start); + return std::vector(data_.begin() + start, + data_.begin() + start + size); + } + + std::vector data_; + EndsleyPatchWriter patch_writer_{&data_}; +}; + +// Smoke check that a patch includes the new_size and magic header. +TEST_F(EndsleyPatchWriterTest, CreateEmptyPatchTest) { + EXPECT_TRUE(patch_writer_.Init(0)); + EXPECT_TRUE(patch_writer_.Close()); + + // The empty header is set to 24 bytes. + EXPECT_EQ(24U, data_.size()); + + std::vector empty_patch = { + // Magic header. + 'E', 'N', 'D', 'S', 'L', 'E', 'Y', '/', 'B', 'S', 'D', 'I', 'F', 'F', '4', + '3', + // 8 zeros for the |new_size| of zero bytes. + 0, 0, 0, 0, 0, 0, 0, 0}; + EXPECT_EQ(empty_patch, data_); +} + +// Test we generate the right patch when the control, diff and extra stream come +// in the right order. +TEST_F(EndsleyPatchWriterTest, DataInNiceOrderTest) { + auto text = VectorFromString("abcdeFGHIJ"); + EXPECT_TRUE(patch_writer_.Init(10)); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2))); + EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 2)); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 2, 3)); + + // Check that we are actually writing to the output vector as soon as we can. + EXPECT_EQ(24U + 24U + 2U + 3U, data_.size()); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(0, 5, 1024))); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5)); + + EXPECT_TRUE(patch_writer_.Close()); + + // We have a header, 2 control entries and a total of 10 bytes of data. + EXPECT_EQ(24U + 24U * 2 + 10U, data_.size()); + + // Verify that control entry values are encoded properly in little-endian. + EXPECT_EQ((std::vector{10, 0, 0, 0, 0, 0, 0, 0}), + DataSubvector(16U, 8)); // new_size + + // Negative numbers are encoded with the sign bit in the most significant bit + // of the 8-byte number. + EXPECT_EQ((std::vector{2, 0, 0, 0, 0, 0, 0, 0x80}), + DataSubvector(24U + 16, 8)); + + // The second member on the last control entry (1024) encoded in + // little-endian. + EXPECT_EQ((std::vector{0, 4, 0, 0, 0, 0, 0, 0}), + DataSubvector(24U + 24U + 5U + 16U, 8)); + + // Check that the diff and extra data are sent one after the other in the + // right order. + EXPECT_EQ(VectorFromString("abcde"), DataSubvector(24U + 24U, 5)); +} + +// When we send first the diff or extra data it shouldn't be possible to +// write it to the patch, but at the end of the patch we should be able to +// write it all. +TEST_F(EndsleyPatchWriterTest, DataInBadOrderTest) { + auto text = VectorFromString("abcdeFGHIJ"); + EXPECT_TRUE(patch_writer_.Init(10)); + EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 5)); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5)); + + // Writ all the control entries at the end, only the header should have been + // sent so far. + EXPECT_EQ(24U, data_.size()); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2))); + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 1, 1024))); + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(1, 1, 1024))); + + EXPECT_TRUE(patch_writer_.Close()); + + // We have a header, 3 control entries and a total of 10 bytes of data. + EXPECT_EQ(24U + 24U * 3 + 10U, data_.size()); + + // The data from the first and second control entries: + EXPECT_EQ(VectorFromString("abFGH"), DataSubvector(24U + 24U, 5)); + EXPECT_EQ(VectorFromString("cdI"), DataSubvector(24U + 24U * 2 + 5, 3)); + EXPECT_EQ(VectorFromString("eJ"), DataSubvector(24U + 24U * 3 + 8, 2)); +} + +TEST_F(EndsleyPatchWriterTest, FlushOnlyWhenWorthItTest) { + size_t kEntrySize = 1000; // must be even for this test. + size_t kNumEntries = 3000; + size_t kNewSize = kEntrySize * kNumEntries; // 3 MB + + EXPECT_TRUE(patch_writer_.Init(kNewSize)); + // Write all the extra and diff data first. + std::vector zeros(kNewSize / 2, 0); + EXPECT_TRUE(patch_writer_.WriteDiffStream(zeros.data(), zeros.size())); + EXPECT_TRUE(patch_writer_.WriteExtraStream(zeros.data(), zeros.size())); + + // No patch data flushed so far, only the header. + EXPECT_EQ(24U, data_.size()); + + ControlEntry entry(kEntrySize / 2, kEntrySize / 2, -1); + for (size_t i = 0; i < 10; i++) { + EXPECT_TRUE(patch_writer_.AddControlEntry(entry)); + } + + // Even if all the diff and extra data is available and some control entries + // are also available no information should have been flushed yet because we + // don't want the overhead of updating the diff_data_ and extra_data_ vectors. + EXPECT_EQ(24U, data_.size()); + + // Write the remaining entries. + for (size_t i = 0; i < kNumEntries - 10; i++) { + EXPECT_TRUE(patch_writer_.AddControlEntry(entry)); + } + + // Even before Close() is called, we have enough control entries to make it + // worth it calling flush at some point. + EXPECT_LT(24U, data_.size()); + + EXPECT_TRUE(patch_writer_.Close()); +} + +} // namespace bsdiff diff --git a/include/bsdiff/patch_writer_factory.h b/include/bsdiff/patch_writer_factory.h index 3e57bcd..2320334 100644 --- a/include/bsdiff/patch_writer_factory.h +++ b/include/bsdiff/patch_writer_factory.h @@ -7,6 +7,7 @@ #include #include +#include #include "bsdiff/patch_writer_interface.h" @@ -17,6 +18,12 @@ namespace bsdiff { std::unique_ptr CreateBsdiffPatchWriter( const std::string& patch_filename); +// Create a patch writer compatible with Android Play Store bsdiff patches, +// uncompressed. The data will be written to the passed |patch| vector, which +// must be valid until Close() is called or this patch is destroyed. +std::unique_ptr CreateEndsleyPatchWriter( + std::vector* patch); + } // namespace bsdiff #endif // _BSDIFF_PATCH_WRITER_FACTORY_H_ diff --git a/patch_writer_factory.cc b/patch_writer_factory.cc index 84cd3dc..d86bafc 100644 --- a/patch_writer_factory.cc +++ b/patch_writer_factory.cc @@ -4,6 +4,7 @@ #include "bsdiff/patch_writer_factory.h" +#include "bsdiff/endsley_patch_writer.h" #include "bsdiff/patch_writer.h" namespace bsdiff { @@ -15,4 +16,9 @@ std::unique_ptr CreateBsdiffPatchWriter( new BsdiffPatchWriter(patch_filename, CompressorType::kBZ2)); } +std::unique_ptr CreateEndsleyPatchWriter( + std::vector* patch) { + return std::unique_ptr(new EndsleyPatchWriter(patch)); +} + } // namespace bsdiff