From 710b3daa482eb100bdaae7ffb46724273bd404d5 Mon Sep 17 00:00:00 2001 From: Alex Deymo Date: Thu, 26 Oct 2017 13:13:28 +0200 Subject: [PATCH] Implement Play Store's "endsley" patching format generator. Android's Play Store uses, among other options, a bsdiff patch format similar to Matthew Endsley's bsdiff implementation. The main difference is that Play Store's version doesn't do any compression in the patch format itself, but instead the compression is handled at a higher level. This CL introduces a new PatchWriterInterface derived class to write to this (uncompressed) format. See https://github.com/mendsley/bsdiff for the original implementation of this format. See also Google's APK patch size estimator for more information on the file-by-file format used by Play Store: https://github.com/googlesamples/apk-patch-size-estimator Bug: 67458097 Test: Added simple unittests for the generated files. Change-Id: I3b048abf2220e2cb3af0a95e27efe1c0f241046a --- Android.bp | 2 + Makefile | 2 + bsdiff.gyp | 2 + endsley_patch_writer.cc | 178 ++++++++++++++++++++++++++ endsley_patch_writer.h | 83 ++++++++++++ endsley_patch_writer_unittest.cc | 158 +++++++++++++++++++++++ include/bsdiff/patch_writer_factory.h | 7 + patch_writer_factory.cc | 6 + 8 files changed, 438 insertions(+) create mode 100644 endsley_patch_writer.cc create mode 100644 endsley_patch_writer.h create mode 100644 endsley_patch_writer_unittest.cc diff --git a/Android.bp b/Android.bp index 4566caa..fc224e9 100644 --- a/Android.bp +++ b/Android.bp @@ -59,6 +59,7 @@ cc_library_static { "compressor_buffer.cc", "compressor_interface.cc", "diff_encoder.cc", + "endsley_patch_writer.cc", "patch_writer.cc", "patch_writer_factory.cc", "split_patch_writer.cc", @@ -109,6 +110,7 @@ cc_test { "bsdiff_unittest.cc", "bspatch_unittest.cc", "diff_encoder_unittest.cc", + "endsley_patch_writer_unittest.cc", "extents_file_unittest.cc", "extents_unittest.cc", "patch_reader_unittest.cc", diff --git a/Makefile b/Makefile index b76e3cb..6a06440 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ bsdiff_src_files := \ compressor_buffer.cc \ compressor_interface.cc \ diff_encoder.cc \ + endsley_patch_writer.cc \ patch_writer.cc \ patch_writer_factory.cc \ split_patch_writer.cc \ @@ -63,6 +64,7 @@ bsdiff_common_unittests := \ bsdiff_unittest.cc \ bspatch_unittest.cc \ diff_encoder_unittest.cc \ + endsley_patch_writer_unittest.cc \ extents_file_unittest.cc \ extents_unittest.cc \ patch_reader_unittest.cc \ diff --git a/bsdiff.gyp b/bsdiff.gyp index 3be7108..14d8c9a 100644 --- a/bsdiff.gyp +++ b/bsdiff.gyp @@ -62,6 +62,7 @@ 'compressor_buffer.cc', 'compressor_interface.cc', 'diff_encoder.cc', + 'endsley_patch_writer.cc', 'patch_writer.cc', 'patch_writer_factory.cc', 'split_patch_writer.cc', @@ -140,6 +141,7 @@ 'bsdiff_unittest.cc', 'bspatch_unittest.cc', 'diff_encoder_unittest.cc', + 'endsley_patch_writer_unittest.cc', 'extents_file_unittest.cc', 'extents_unittest.cc', 'patch_reader_unittest.cc', diff --git a/endsley_patch_writer.cc b/endsley_patch_writer.cc new file mode 100644 index 0000000..4e4a3f1 --- /dev/null +++ b/endsley_patch_writer.cc @@ -0,0 +1,178 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/endsley_patch_writer.h" + +#include + +#include + +#include "bsdiff/logging.h" + +using std::endl; + +namespace { + +constexpr uint8_t kEndsleyMagicHeader[] = "ENDSLEY/BSDIFF43"; + +void EncodeInt64(int64_t x, uint8_t* buf) { + uint64_t y = x < 0 ? (1ULL << 63ULL) - x : x; + for (int i = 0; i < 8; ++i) { + buf[i] = y & 0xff; + y /= 256; + } +} + +// The minimum size that we would consider flushing out. +constexpr size_t kMinimumFlushSize = 1024 * 1024; // 1 MiB + +} // namespace + +namespace bsdiff { + +bool EndsleyPatchWriter::Init(size_t new_size) { + // The patch is uncompressed and it will need exactly: + // new_size + 24 * len(control_entries) + sizeof(header) + // We don't know the length of the control entries yet, but we can reserve + // enough space to hold at least |new_size|. + patch_->clear(); + patch_->reserve(new_size); + + // Header is the magic followed by the new length. + uint8_t header[24]; + memcpy(header, kEndsleyMagicHeader, 16); + EncodeInt64(new_size, header + 16); + EmitBuffer(header, sizeof(header)); + return true; +} + +bool EndsleyPatchWriter::WriteDiffStream(const uint8_t* data, size_t size) { + if (!size) + return true; + // Speed-up the common case where the diff stream data is added right after + // the control entry that refers to it. + if (control_.empty() && pending_diff_ >= size) { + pending_diff_ -= size; + EmitBuffer(data, size); + return true; + } + + diff_data_.insert(diff_data_.end(), data, data + size); + return true; +} + +bool EndsleyPatchWriter::WriteExtraStream(const uint8_t* data, size_t size) { + if (!size) + return true; + // Speed-up the common case where the extra stream data is added right after + // the diff stream data and the control entry that refers to it. Note that + // the diff data comes first so we need to make sure it is all out. + if (control_.empty() && !pending_diff_ && pending_extra_ >= size) { + pending_extra_ -= size; + EmitBuffer(data, size); + return true; + } + + extra_data_.insert(extra_data_.end(), data, data + size); + return true; +} + +bool EndsleyPatchWriter::AddControlEntry(const ControlEntry& entry) { + // Speed-up the common case where the control entry is added when there's + // nothing else pending. + if (control_.empty() && diff_data_.empty() && extra_data_.empty() && + !pending_diff_ && !pending_extra_) { + pending_diff_ = entry.diff_size; + pending_extra_ = entry.extra_size; + EmitControlEntry(entry); + return true; + } + + control_.push_back(entry); + pending_control_data_ += entry.diff_size + entry.extra_size; + + // Check whether it is worth Flushing the entries now that the we have more + // control entries. We need control entries to write enough output data, and + // we need that output data to be at least 50% of the available diff and extra + // data. This last requirement is to reduce the overhead of removing the + // flushed data. + if (pending_control_data_ > kMinimumFlushSize && + (diff_data_.size() + extra_data_.size()) / 2 <= pending_control_data_) { + Flush(); + } + + return true; +} + +bool EndsleyPatchWriter::Close() { + // Flush any pending data. + Flush(); + + if (pending_diff_ || pending_extra_ || !control_.empty()) { + LOG(ERROR) << "Insufficient data sent to diff/extra streams" << endl; + return false; + } + + if (!diff_data_.empty() || !extra_data_.empty()) { + LOG(ERROR) << "Pending data to diff/extra not flushed out on Close()" + << endl; + return false; + } + return true; +} + +void EndsleyPatchWriter::EmitControlEntry(const ControlEntry& entry) { + // Generate the 24 byte control entry. + uint8_t buf[24]; + EncodeInt64(entry.diff_size, buf); + EncodeInt64(entry.extra_size, buf + 8); + EncodeInt64(entry.offset_increment, buf + 16); + EmitBuffer(buf, sizeof(buf)); +} + +void EndsleyPatchWriter::EmitBuffer(const uint8_t* data, size_t size) { + patch_->insert(patch_->end(), data, data + size); +} + +void EndsleyPatchWriter::Flush() { + size_t used_diff = 0; + size_t used_extra = 0; + size_t used_control = 0; + do { + if (!pending_diff_ && !pending_extra_ && used_control < control_.size()) { + // We can emit a control entry in these conditions. + const ControlEntry& entry = control_[used_control]; + used_control++; + + pending_diff_ = entry.diff_size; + pending_extra_ = entry.extra_size; + pending_control_data_ -= entry.extra_size + entry.diff_size; + EmitControlEntry(entry); + } + + if (pending_diff_) { + size_t diff_size = std::min(diff_data_.size() - used_diff, pending_diff_); + EmitBuffer(diff_data_.data() + used_diff, diff_size); + pending_diff_ -= diff_size; + used_diff += diff_size; + } + + if (!pending_diff_ && pending_extra_) { + size_t extra_size = + std::min(extra_data_.size() - used_extra, pending_extra_); + EmitBuffer(extra_data_.data() + used_extra, extra_size); + pending_extra_ -= extra_size; + used_extra += extra_size; + } + } while (!pending_diff_ && !pending_extra_ && used_control < control_.size()); + + if (used_diff) + diff_data_.erase(diff_data_.begin(), diff_data_.begin() + used_diff); + if (used_extra) + extra_data_.erase(extra_data_.begin(), extra_data_.begin() + used_extra); + if (used_control) + control_.erase(control_.begin(), control_.begin() + used_control); +} + +} // namespace bsdiff diff --git a/endsley_patch_writer.h b/endsley_patch_writer.h new file mode 100644 index 0000000..1840d8d --- /dev/null +++ b/endsley_patch_writer.h @@ -0,0 +1,83 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef _BSDIFF_ENDSLEY_PATCH_WRITER_H_ +#define _BSDIFF_ENDSLEY_PATCH_WRITER_H_ + +#include +#include + +#include "bsdiff/patch_writer_interface.h" + +namespace bsdiff { + +// A PatchWriterInterface class compatible with the format used by Android Play +// Store's bsdiff implementation, which is based on Matthew Endsley's bsdiff +// implementation. See https://github.com/mendsley/bsdiff for the original +// implementation of this format. See also Google's APK patch size estimator for +// more information on the file-by-file format used by Play Store: +// https://github.com/googlesamples/apk-patch-size-estimator + +// This format, identified by the "ENDSLEY/BSDIFF43" magic string, uses a single +// stream with the control entries, diff data and extra data interleaved. After +// the header, each Control Entry is stored in 24 bytes followed by the diff +// stream data for that entry only, and then followed by the extra stream data +// for that entry only. The format doesn't handle the compression of the data, +// instead, the whole file (including the magic string) is compressed with any +// compression algorithm. + +// This format is easier to parse and allows the patch to be streamed, but by +// mixing the diff and extra data into the same compression context offers a +// slightly worse compression ratio (about 3.5% compared to upstream's format). + +class EndsleyPatchWriter : public PatchWriterInterface { + public: + // Create the patch writer that will write the data to the passed vector + // |patch|, resizing it as needed. The |patch| vector must be valid until + // Close() is called or this patch is destroyed. + explicit EndsleyPatchWriter(std::vector* patch) : patch_(patch) {} + + // PatchWriterInterface overrides. + bool Init(size_t new_size) override; + bool WriteDiffStream(const uint8_t* data, size_t size) override; + bool WriteExtraStream(const uint8_t* data, size_t size) override; + bool AddControlEntry(const ControlEntry& entry) override; + bool Close() override; + + private: + // Emit at the end of the |patch_| vector the passed control entry. + void EmitControlEntry(const ControlEntry& entry); + + // Emit at the end of the |patch_| vector the passed buffer. + void EmitBuffer(const uint8_t* data, size_t size); + + // Flush as much as possible of the pending data. + void Flush(); + + // The vector we are writing to, owned by the caller. + std::vector* patch_; + + // The pending diff and extra data to be encoded in the file. These vectors + // would not be used whenever is possible to the data directly to the patch_ + // vector; namely when the control, diff and extra stream data are provided in + // that order for each control entry. + std::vector diff_data_; + std::vector extra_data_; + std::vector control_; + + // Defined as the sum of all the diff_size and extra_size values in + // |control_|. This is used to determine whether it is worth Flushing the + // pending data. + size_t pending_control_data_{0}; + + // Number of bytes in the diff and extra stream that are pending in the + // last control entry encoded in the |patch_|. If both are zero the last + // control entry was completely emitted. + size_t pending_diff_{0}; + size_t pending_extra_{0}; +}; + +} // namespace bsdiff + +#endif // _BSDIFF_ENDSLEY_PATCH_WRITER_H_ diff --git a/endsley_patch_writer_unittest.cc b/endsley_patch_writer_unittest.cc new file mode 100644 index 0000000..29f404c --- /dev/null +++ b/endsley_patch_writer_unittest.cc @@ -0,0 +1,158 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "bsdiff/endsley_patch_writer.h" + +#include + +#include + +namespace { + +std::vector VectorFromString(const std::string& s) { + return std::vector(s.data(), s.data() + s.size()); +} + +} // namespace + +namespace bsdiff { + +class EndsleyPatchWriterTest : public testing::Test { + protected: + // Return a subvector from |data_| starting at |start| of size at most |size|. + std::vector DataSubvector(size_t start, size_t size) { + if (start > data_.size()) + return std::vector(); + + size = std::min(size, data_.size() - start); + return std::vector(data_.begin() + start, + data_.begin() + start + size); + } + + std::vector data_; + EndsleyPatchWriter patch_writer_{&data_}; +}; + +// Smoke check that a patch includes the new_size and magic header. +TEST_F(EndsleyPatchWriterTest, CreateEmptyPatchTest) { + EXPECT_TRUE(patch_writer_.Init(0)); + EXPECT_TRUE(patch_writer_.Close()); + + // The empty header is set to 24 bytes. + EXPECT_EQ(24U, data_.size()); + + std::vector empty_patch = { + // Magic header. + 'E', 'N', 'D', 'S', 'L', 'E', 'Y', '/', 'B', 'S', 'D', 'I', 'F', 'F', '4', + '3', + // 8 zeros for the |new_size| of zero bytes. + 0, 0, 0, 0, 0, 0, 0, 0}; + EXPECT_EQ(empty_patch, data_); +} + +// Test we generate the right patch when the control, diff and extra stream come +// in the right order. +TEST_F(EndsleyPatchWriterTest, DataInNiceOrderTest) { + auto text = VectorFromString("abcdeFGHIJ"); + EXPECT_TRUE(patch_writer_.Init(10)); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2))); + EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 2)); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 2, 3)); + + // Check that we are actually writing to the output vector as soon as we can. + EXPECT_EQ(24U + 24U + 2U + 3U, data_.size()); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(0, 5, 1024))); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5)); + + EXPECT_TRUE(patch_writer_.Close()); + + // We have a header, 2 control entries and a total of 10 bytes of data. + EXPECT_EQ(24U + 24U * 2 + 10U, data_.size()); + + // Verify that control entry values are encoded properly in little-endian. + EXPECT_EQ((std::vector{10, 0, 0, 0, 0, 0, 0, 0}), + DataSubvector(16U, 8)); // new_size + + // Negative numbers are encoded with the sign bit in the most significant bit + // of the 8-byte number. + EXPECT_EQ((std::vector{2, 0, 0, 0, 0, 0, 0, 0x80}), + DataSubvector(24U + 16, 8)); + + // The second member on the last control entry (1024) encoded in + // little-endian. + EXPECT_EQ((std::vector{0, 4, 0, 0, 0, 0, 0, 0}), + DataSubvector(24U + 24U + 5U + 16U, 8)); + + // Check that the diff and extra data are sent one after the other in the + // right order. + EXPECT_EQ(VectorFromString("abcde"), DataSubvector(24U + 24U, 5)); +} + +// When we send first the diff or extra data it shouldn't be possible to +// write it to the patch, but at the end of the patch we should be able to +// write it all. +TEST_F(EndsleyPatchWriterTest, DataInBadOrderTest) { + auto text = VectorFromString("abcdeFGHIJ"); + EXPECT_TRUE(patch_writer_.Init(10)); + EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 5)); + EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5)); + + // Writ all the control entries at the end, only the header should have been + // sent so far. + EXPECT_EQ(24U, data_.size()); + + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2))); + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 1, 1024))); + EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(1, 1, 1024))); + + EXPECT_TRUE(patch_writer_.Close()); + + // We have a header, 3 control entries and a total of 10 bytes of data. + EXPECT_EQ(24U + 24U * 3 + 10U, data_.size()); + + // The data from the first and second control entries: + EXPECT_EQ(VectorFromString("abFGH"), DataSubvector(24U + 24U, 5)); + EXPECT_EQ(VectorFromString("cdI"), DataSubvector(24U + 24U * 2 + 5, 3)); + EXPECT_EQ(VectorFromString("eJ"), DataSubvector(24U + 24U * 3 + 8, 2)); +} + +TEST_F(EndsleyPatchWriterTest, FlushOnlyWhenWorthItTest) { + size_t kEntrySize = 1000; // must be even for this test. + size_t kNumEntries = 3000; + size_t kNewSize = kEntrySize * kNumEntries; // 3 MB + + EXPECT_TRUE(patch_writer_.Init(kNewSize)); + // Write all the extra and diff data first. + std::vector zeros(kNewSize / 2, 0); + EXPECT_TRUE(patch_writer_.WriteDiffStream(zeros.data(), zeros.size())); + EXPECT_TRUE(patch_writer_.WriteExtraStream(zeros.data(), zeros.size())); + + // No patch data flushed so far, only the header. + EXPECT_EQ(24U, data_.size()); + + ControlEntry entry(kEntrySize / 2, kEntrySize / 2, -1); + for (size_t i = 0; i < 10; i++) { + EXPECT_TRUE(patch_writer_.AddControlEntry(entry)); + } + + // Even if all the diff and extra data is available and some control entries + // are also available no information should have been flushed yet because we + // don't want the overhead of updating the diff_data_ and extra_data_ vectors. + EXPECT_EQ(24U, data_.size()); + + // Write the remaining entries. + for (size_t i = 0; i < kNumEntries - 10; i++) { + EXPECT_TRUE(patch_writer_.AddControlEntry(entry)); + } + + // Even before Close() is called, we have enough control entries to make it + // worth it calling flush at some point. + EXPECT_LT(24U, data_.size()); + + EXPECT_TRUE(patch_writer_.Close()); +} + +} // namespace bsdiff diff --git a/include/bsdiff/patch_writer_factory.h b/include/bsdiff/patch_writer_factory.h index 3e57bcd..2320334 100644 --- a/include/bsdiff/patch_writer_factory.h +++ b/include/bsdiff/patch_writer_factory.h @@ -7,6 +7,7 @@ #include #include +#include #include "bsdiff/patch_writer_interface.h" @@ -17,6 +18,12 @@ namespace bsdiff { std::unique_ptr CreateBsdiffPatchWriter( const std::string& patch_filename); +// Create a patch writer compatible with Android Play Store bsdiff patches, +// uncompressed. The data will be written to the passed |patch| vector, which +// must be valid until Close() is called or this patch is destroyed. +std::unique_ptr CreateEndsleyPatchWriter( + std::vector* patch); + } // namespace bsdiff #endif // _BSDIFF_PATCH_WRITER_FACTORY_H_ diff --git a/patch_writer_factory.cc b/patch_writer_factory.cc index 84cd3dc..d86bafc 100644 --- a/patch_writer_factory.cc +++ b/patch_writer_factory.cc @@ -4,6 +4,7 @@ #include "bsdiff/patch_writer_factory.h" +#include "bsdiff/endsley_patch_writer.h" #include "bsdiff/patch_writer.h" namespace bsdiff { @@ -15,4 +16,9 @@ std::unique_ptr CreateBsdiffPatchWriter( new BsdiffPatchWriter(patch_filename, CompressorType::kBZ2)); } +std::unique_ptr CreateEndsleyPatchWriter( + std::vector* patch) { + return std::unique_ptr(new EndsleyPatchWriter(patch)); +} + } // namespace bsdiff