Merge "Implement Play Store's "endsley" patching format generator."

am: b2af52d1eb

Change-Id: I811a49ba1e1a6bcab7c45b99f4c37ece6151b477

Android.bp

@@ -59,6 +59,7 @@ cc_library_static {
         "compressor_buffer.cc",
         "compressor_interface.cc",
         "diff_encoder.cc",
+        "endsley_patch_writer.cc",
         "patch_writer.cc",
         "patch_writer_factory.cc",
         "split_patch_writer.cc",
@@ -109,6 +110,7 @@ cc_test {
         "bsdiff_unittest.cc",
         "bspatch_unittest.cc",
         "diff_encoder_unittest.cc",
+        "endsley_patch_writer_unittest.cc",
         "extents_file_unittest.cc",
         "extents_unittest.cc",
         "patch_reader_unittest.cc",

Makefile

@@ -39,6 +39,7 @@ bsdiff_src_files := \
     compressor_buffer.cc \
     compressor_interface.cc \
     diff_encoder.cc \
+    endsley_patch_writer.cc \
     patch_writer.cc \
     patch_writer_factory.cc \
     split_patch_writer.cc \
@@ -63,6 +64,7 @@ bsdiff_common_unittests := \
     bsdiff_unittest.cc \
     bspatch_unittest.cc \
     diff_encoder_unittest.cc \
+    endsley_patch_writer_unittest.cc \
     extents_file_unittest.cc \
     extents_unittest.cc \
     patch_reader_unittest.cc \

bsdiff.gyp

@@ -62,6 +62,7 @@
         'compressor_buffer.cc',
         'compressor_interface.cc',
         'diff_encoder.cc',
+        'endsley_patch_writer.cc',
         'patch_writer.cc',
         'patch_writer_factory.cc',
         'split_patch_writer.cc',
@@ -140,6 +141,7 @@
             'bsdiff_unittest.cc',
             'bspatch_unittest.cc',
             'diff_encoder_unittest.cc',
+            'endsley_patch_writer_unittest.cc',
             'extents_file_unittest.cc',
             'extents_unittest.cc',
             'patch_reader_unittest.cc',

endsley_patch_writer.cc

@@ -0,0 +1,178 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "bsdiff/endsley_patch_writer.h"
+
+#include <string.h>
+
+#include <algorithm>
+
+#include "bsdiff/logging.h"
+
+using std::endl;
+
+namespace {
+
+constexpr uint8_t kEndsleyMagicHeader[] = "ENDSLEY/BSDIFF43";
+
+void EncodeInt64(int64_t x, uint8_t* buf) {
+  uint64_t y = x < 0 ? (1ULL << 63ULL) - x : x;
+  for (int i = 0; i < 8; ++i) {
+    buf[i] = y & 0xff;
+    y /= 256;
+  }
+}
+
+// The minimum size that we would consider flushing out.
+constexpr size_t kMinimumFlushSize = 1024 * 1024;  // 1 MiB
+
+}  // namespace
+
+namespace bsdiff {
+
+bool EndsleyPatchWriter::Init(size_t new_size) {
+  // The patch is uncompressed and it will need exactly:
+  //   new_size + 24 * len(control_entries) + sizeof(header)
+  // We don't know the length of the control entries yet, but we can reserve
+  // enough space to hold at least |new_size|.
+  patch_->clear();
+  patch_->reserve(new_size);
+
+  // Header is the magic followed by the new length.
+  uint8_t header[24];
+  memcpy(header, kEndsleyMagicHeader, 16);
+  EncodeInt64(new_size, header + 16);
+  EmitBuffer(header, sizeof(header));
+  return true;
+}
+
+bool EndsleyPatchWriter::WriteDiffStream(const uint8_t* data, size_t size) {
+  if (!size)
+    return true;
+  // Speed-up the common case where the diff stream data is added right after
+  // the control entry that refers to it.
+  if (control_.empty() && pending_diff_ >= size) {
+    pending_diff_ -= size;
+    EmitBuffer(data, size);
+    return true;
+  }
+
+  diff_data_.insert(diff_data_.end(), data, data + size);
+  return true;
+}
+
+bool EndsleyPatchWriter::WriteExtraStream(const uint8_t* data, size_t size) {
+  if (!size)
+    return true;
+  // Speed-up the common case where the extra stream data is added right after
+  // the diff stream data and the control entry that refers to it. Note that
+  // the diff data comes first so we need to make sure it is all out.
+  if (control_.empty() && !pending_diff_ && pending_extra_ >= size) {
+    pending_extra_ -= size;
+    EmitBuffer(data, size);
+    return true;
+  }
+
+  extra_data_.insert(extra_data_.end(), data, data + size);
+  return true;
+}
+
+bool EndsleyPatchWriter::AddControlEntry(const ControlEntry& entry) {
+  // Speed-up the common case where the control entry is added when there's
+  // nothing else pending.
+  if (control_.empty() && diff_data_.empty() && extra_data_.empty() &&
+      !pending_diff_ && !pending_extra_) {
+    pending_diff_ = entry.diff_size;
+    pending_extra_ = entry.extra_size;
+    EmitControlEntry(entry);
+    return true;
+  }
+
+  control_.push_back(entry);
+  pending_control_data_ += entry.diff_size + entry.extra_size;
+
+  // Check whether it is worth Flushing the entries now that the we have more
+  // control entries. We need control entries to write enough output data, and
+  // we need that output data to be at least 50% of the available diff and extra
+  // data. This last requirement is to reduce the overhead of removing the
+  // flushed data.
+  if (pending_control_data_ > kMinimumFlushSize &&
+      (diff_data_.size() + extra_data_.size()) / 2 <= pending_control_data_) {
+    Flush();
+  }
+
+  return true;
+}
+
+bool EndsleyPatchWriter::Close() {
+  // Flush any pending data.
+  Flush();
+
+  if (pending_diff_ || pending_extra_ || !control_.empty()) {
+    LOG(ERROR) << "Insufficient data sent to diff/extra streams" << endl;
+    return false;
+  }
+
+  if (!diff_data_.empty() || !extra_data_.empty()) {
+    LOG(ERROR) << "Pending data to diff/extra not flushed out on Close()"
+               << endl;
+    return false;
+  }
+  return true;
+}
+
+void EndsleyPatchWriter::EmitControlEntry(const ControlEntry& entry) {
+  // Generate the 24 byte control entry.
+  uint8_t buf[24];
+  EncodeInt64(entry.diff_size, buf);
+  EncodeInt64(entry.extra_size, buf + 8);
+  EncodeInt64(entry.offset_increment, buf + 16);
+  EmitBuffer(buf, sizeof(buf));
+}
+
+void EndsleyPatchWriter::EmitBuffer(const uint8_t* data, size_t size) {
+  patch_->insert(patch_->end(), data, data + size);
+}
+
+void EndsleyPatchWriter::Flush() {
+  size_t used_diff = 0;
+  size_t used_extra = 0;
+  size_t used_control = 0;
+  do {
+    if (!pending_diff_ && !pending_extra_ && used_control < control_.size()) {
+      // We can emit a control entry in these conditions.
+      const ControlEntry& entry = control_[used_control];
+      used_control++;
+
+      pending_diff_ = entry.diff_size;
+      pending_extra_ = entry.extra_size;
+      pending_control_data_ -= entry.extra_size + entry.diff_size;
+      EmitControlEntry(entry);
+    }
+
+    if (pending_diff_) {
+      size_t diff_size = std::min(diff_data_.size() - used_diff, pending_diff_);
+      EmitBuffer(diff_data_.data() + used_diff, diff_size);
+      pending_diff_ -= diff_size;
+      used_diff += diff_size;
+    }
+
+    if (!pending_diff_ && pending_extra_) {
+      size_t extra_size =
+          std::min(extra_data_.size() - used_extra, pending_extra_);
+      EmitBuffer(extra_data_.data() + used_extra, extra_size);
+      pending_extra_ -= extra_size;
+      used_extra += extra_size;
+    }
+  } while (!pending_diff_ && !pending_extra_ && used_control < control_.size());
+
+  if (used_diff)
+    diff_data_.erase(diff_data_.begin(), diff_data_.begin() + used_diff);
+  if (used_extra)
+    extra_data_.erase(extra_data_.begin(), extra_data_.begin() + used_extra);
+  if (used_control)
+    control_.erase(control_.begin(), control_.begin() + used_control);
+}
+
+}  // namespace bsdiff

endsley_patch_writer.h

@@ -0,0 +1,83 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef _BSDIFF_ENDSLEY_PATCH_WRITER_H_
+#define _BSDIFF_ENDSLEY_PATCH_WRITER_H_
+
+#include <string>
+#include <vector>
+
+#include "bsdiff/patch_writer_interface.h"
+
+namespace bsdiff {
+
+// A PatchWriterInterface class compatible with the format used by Android Play
+// Store's bsdiff implementation, which is based on Matthew Endsley's bsdiff
+// implementation. See https://github.com/mendsley/bsdiff for the original
+// implementation of this format. See also Google's APK patch size estimator for
+// more information on the file-by-file format used by Play Store:
+// https://github.com/googlesamples/apk-patch-size-estimator
+
+// This format, identified by the "ENDSLEY/BSDIFF43" magic string, uses a single
+// stream with the control entries, diff data and extra data interleaved. After
+// the header, each Control Entry is stored in 24 bytes followed by the diff
+// stream data for that entry only, and then followed by the extra stream data
+// for that entry only. The format doesn't handle the compression of the data,
+// instead, the whole file (including the magic string) is compressed with any
+// compression algorithm.
+
+// This format is easier to parse and allows the patch to be streamed, but by
+// mixing the diff and extra data into the same compression context offers a
+// slightly worse compression ratio (about 3.5% compared to upstream's format).
+
+class EndsleyPatchWriter : public PatchWriterInterface {
+ public:
+  // Create the patch writer that will write the data to the passed vector
+  // |patch|, resizing it as needed. The |patch| vector must be valid until
+  // Close() is called or this patch is destroyed.
+  explicit EndsleyPatchWriter(std::vector<uint8_t>* patch) : patch_(patch) {}
+
+  // PatchWriterInterface overrides.
+  bool Init(size_t new_size) override;
+  bool WriteDiffStream(const uint8_t* data, size_t size) override;
+  bool WriteExtraStream(const uint8_t* data, size_t size) override;
+  bool AddControlEntry(const ControlEntry& entry) override;
+  bool Close() override;
+
+ private:
+  // Emit at the end of the |patch_| vector the passed control entry.
+  void EmitControlEntry(const ControlEntry& entry);
+
+  // Emit at the end of the |patch_| vector the passed buffer.
+  void EmitBuffer(const uint8_t* data, size_t size);
+
+  // Flush as much as possible of the pending data.
+  void Flush();
+
+  // The vector we are writing to, owned by the caller.
+  std::vector<uint8_t>* patch_;
+
+  // The pending diff and extra data to be encoded in the file. These vectors
+  // would not be used whenever is possible to the data directly to the patch_
+  // vector; namely when the control, diff and extra stream data are provided in
+  // that order for each control entry.
+  std::vector<uint8_t> diff_data_;
+  std::vector<uint8_t> extra_data_;
+  std::vector<ControlEntry> control_;
+
+  // Defined as the sum of all the diff_size and extra_size values in
+  // |control_|. This is used to determine whether it is worth Flushing the
+  // pending data.
+  size_t pending_control_data_{0};
+
+  // Number of bytes in the diff and extra stream that are pending in the
+  // last control entry encoded in the |patch_|. If both are zero the last
+  // control entry was completely emitted.
+  size_t pending_diff_{0};
+  size_t pending_extra_{0};
+};
+
+}  // namespace bsdiff
+
+#endif  // _BSDIFF_ENDSLEY_PATCH_WRITER_H_

endsley_patch_writer_unittest.cc

@@ -0,0 +1,158 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "bsdiff/endsley_patch_writer.h"
+
+#include <algorithm>
+
+#include <gtest/gtest.h>
+
+namespace {
+
+std::vector<uint8_t> VectorFromString(const std::string& s) {
+  return std::vector<uint8_t>(s.data(), s.data() + s.size());
+}
+
+}  // namespace
+
+namespace bsdiff {
+
+class EndsleyPatchWriterTest : public testing::Test {
+ protected:
+  // Return a subvector from |data_| starting at |start| of size at most |size|.
+  std::vector<uint8_t> DataSubvector(size_t start, size_t size) {
+    if (start > data_.size())
+      return std::vector<uint8_t>();
+
+    size = std::min(size, data_.size() - start);
+    return std::vector<uint8_t>(data_.begin() + start,
+                                data_.begin() + start + size);
+  }
+
+  std::vector<uint8_t> data_;
+  EndsleyPatchWriter patch_writer_{&data_};
+};
+
+// Smoke check that a patch includes the new_size and magic header.
+TEST_F(EndsleyPatchWriterTest, CreateEmptyPatchTest) {
+  EXPECT_TRUE(patch_writer_.Init(0));
+  EXPECT_TRUE(patch_writer_.Close());
+
+  // The empty header is set to 24 bytes.
+  EXPECT_EQ(24U, data_.size());
+
+  std::vector<uint8_t> empty_patch = {
+      // Magic header.
+      'E', 'N', 'D', 'S', 'L', 'E', 'Y', '/', 'B', 'S', 'D', 'I', 'F', 'F', '4',
+      '3',
+      // 8 zeros for the |new_size| of zero bytes.
+      0, 0, 0, 0, 0, 0, 0, 0};
+  EXPECT_EQ(empty_patch, data_);
+}
+
+// Test we generate the right patch when the control, diff and extra stream come
+// in the right order.
+TEST_F(EndsleyPatchWriterTest, DataInNiceOrderTest) {
+  auto text = VectorFromString("abcdeFGHIJ");
+  EXPECT_TRUE(patch_writer_.Init(10));
+
+  EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2)));
+  EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 2));
+  EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 2, 3));
+
+  // Check that we are actually writing to the output vector as soon as we can.
+  EXPECT_EQ(24U + 24U + 2U + 3U, data_.size());
+
+  EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(0, 5, 1024)));
+  EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5));
+
+  EXPECT_TRUE(patch_writer_.Close());
+
+  // We have a header, 2 control entries and a total of 10 bytes of data.
+  EXPECT_EQ(24U + 24U * 2 + 10U, data_.size());
+
+  // Verify that control entry values are encoded properly in little-endian.
+  EXPECT_EQ((std::vector<uint8_t>{10, 0, 0, 0, 0, 0, 0, 0}),
+            DataSubvector(16U, 8));  // new_size
+
+  // Negative numbers are encoded with the sign bit in the most significant bit
+  // of the 8-byte number.
+  EXPECT_EQ((std::vector<uint8_t>{2, 0, 0, 0, 0, 0, 0, 0x80}),
+            DataSubvector(24U + 16, 8));
+
+  // The second member on the last control entry (1024) encoded in
+  // little-endian.
+  EXPECT_EQ((std::vector<uint8_t>{0, 4, 0, 0, 0, 0, 0, 0}),
+            DataSubvector(24U + 24U + 5U + 16U, 8));
+
+  // Check that the diff and extra data are sent one after the other in the
+  // right order.
+  EXPECT_EQ(VectorFromString("abcde"), DataSubvector(24U + 24U, 5));
+}
+
+// When we send first the diff or extra data it shouldn't be possible to
+// write it to the patch, but at the end of the patch we should be able to
+// write it all.
+TEST_F(EndsleyPatchWriterTest, DataInBadOrderTest) {
+  auto text = VectorFromString("abcdeFGHIJ");
+  EXPECT_TRUE(patch_writer_.Init(10));
+  EXPECT_TRUE(patch_writer_.WriteDiffStream(text.data(), 5));
+  EXPECT_TRUE(patch_writer_.WriteExtraStream(text.data() + 5, 5));
+
+  // Writ all the control entries at the end, only the header should have been
+  // sent so far.
+  EXPECT_EQ(24U, data_.size());
+
+  EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 3, -2)));
+  EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(2, 1, 1024)));
+  EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(1, 1, 1024)));
+
+  EXPECT_TRUE(patch_writer_.Close());
+
+  // We have a header, 3 control entries and a total of 10 bytes of data.
+  EXPECT_EQ(24U + 24U * 3 + 10U, data_.size());
+
+  // The data from the first and second control entries:
+  EXPECT_EQ(VectorFromString("abFGH"), DataSubvector(24U + 24U, 5));
+  EXPECT_EQ(VectorFromString("cdI"), DataSubvector(24U + 24U * 2 + 5, 3));
+  EXPECT_EQ(VectorFromString("eJ"), DataSubvector(24U + 24U * 3 + 8, 2));
+}
+
+TEST_F(EndsleyPatchWriterTest, FlushOnlyWhenWorthItTest) {
+  size_t kEntrySize = 1000;  // must be even for this test.
+  size_t kNumEntries = 3000;
+  size_t kNewSize = kEntrySize * kNumEntries;  // 3 MB
+
+  EXPECT_TRUE(patch_writer_.Init(kNewSize));
+  // Write all the extra and diff data first.
+  std::vector<uint8_t> zeros(kNewSize / 2, 0);
+  EXPECT_TRUE(patch_writer_.WriteDiffStream(zeros.data(), zeros.size()));
+  EXPECT_TRUE(patch_writer_.WriteExtraStream(zeros.data(), zeros.size()));
+
+  // No patch data flushed so far, only the header.
+  EXPECT_EQ(24U, data_.size());
+
+  ControlEntry entry(kEntrySize / 2, kEntrySize / 2, -1);
+  for (size_t i = 0; i < 10; i++) {
+    EXPECT_TRUE(patch_writer_.AddControlEntry(entry));
+  }
+
+  // Even if all the diff and extra data is available and some control entries
+  // are also available no information should have been flushed yet because we
+  // don't want the overhead of updating the diff_data_ and extra_data_ vectors.
+  EXPECT_EQ(24U, data_.size());
+
+  // Write the remaining entries.
+  for (size_t i = 0; i < kNumEntries - 10; i++) {
+    EXPECT_TRUE(patch_writer_.AddControlEntry(entry));
+  }
+
+  // Even before Close() is called, we have enough control entries to make it
+  // worth it calling flush at some point.
+  EXPECT_LT(24U, data_.size());
+
+  EXPECT_TRUE(patch_writer_.Close());
+}
+
+}  // namespace bsdiff

include/bsdiff/patch_writer_factory.h

@@ -7,6 +7,7 @@
 
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "bsdiff/patch_writer_interface.h"
 
@@ -17,6 +18,12 @@ namespace bsdiff {
 std::unique_ptr<PatchWriterInterface> CreateBsdiffPatchWriter(
     const std::string& patch_filename);
 
+// Create a patch writer compatible with Android Play Store bsdiff patches,
+// uncompressed. The data will be written to the passed |patch| vector, which
+// must be valid until Close() is called or this patch is destroyed.
+std::unique_ptr<PatchWriterInterface> CreateEndsleyPatchWriter(
+    std::vector<uint8_t>* patch);
+
 }  // namespace bsdiff
 
 #endif  // _BSDIFF_PATCH_WRITER_FACTORY_H_

patch_writer_factory.cc

@@ -4,6 +4,7 @@
 
 #include "bsdiff/patch_writer_factory.h"
 
+#include "bsdiff/endsley_patch_writer.h"
 #include "bsdiff/patch_writer.h"
 
 namespace bsdiff {
@@ -15,4 +16,9 @@ std::unique_ptr<PatchWriterInterface> CreateBsdiffPatchWriter(
       new BsdiffPatchWriter(patch_filename, CompressorType::kBZ2));
 }
 
+std::unique_ptr<PatchWriterInterface> CreateEndsleyPatchWriter(
+    std::vector<uint8_t>* patch) {
+  return std::unique_ptr<PatchWriterInterface>(new EndsleyPatchWriter(patch));
+}
+
 }  // namespace bsdiff