Merge "Reduce PatchWriterInterface functionality." am: f356f5013f

am: ac6eabf60f

Change-Id: Ia564926520f2a2e3df20e99b3d3717ea47219277
This commit is contained in:
Alex Deymo authored and android-build-merger committed 2017-10-09 17:44:01 +00:00
commit 13f69368da
12 files changed
+290 -107

No files matched your search

+2
View File
@@ -57,6 +57,7 @@ cc_library_static {
srcs: [
"bsdiff.cc",
"bz2_compressor.cc",
"diff_encoder.cc",
"patch_writer.cc",
],
static_libs: [
@@ -99,6 +100,7 @@ cc_test {
srcs: [
"bsdiff_unittest.cc",
"bspatch_unittest.cc",
"diff_encoder_unittest.cc",
"extents_file_unittest.cc",
"extents_unittest.cc",
"patch_writer_unittest.cc",
+2
View File
@@ -35,6 +35,7 @@ clean:
bsdiff_src_files := \
bsdiff.cc \
bz2_compressor.cc \
diff_encoder.cc \
patch_writer.cc
# "bspatch" program.
@@ -51,6 +52,7 @@ bspatch_src_files := \
bsdiff_common_unittests := \
bsdiff_unittest.cc \
bspatch_unittest.cc \
diff_encoder_unittest.cc \
extents_file_unittest.cc \
extents_unittest.cc \
patch_writer_unittest.cc \
+6 -4
View File
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05
#include <algorithm>
#include "bsdiff/diff_encoder.h"
#include "bsdiff/patch_writer.h"
namespace bsdiff {
@@ -149,9 +150,10 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
*I_cache = I;
}
/* Initialize the patch file */
if (!patch->InitializeBuffers(old_buf, oldsize, new_buf, newsize))
/* Initialize the patch file encoder */
if (!patch->Init())
return 1;
DiffEncoder diff_encoder(patch, old_buf, oldsize, new_buf, newsize);
/* Compute the differences, writing ctrl as we go */
scan=0;len=0;
@@ -229,7 +231,7 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
lenb-=lens;
};
if (!patch->AddControlEntry(
if (!diff_encoder.AddControlEntry(
ControlEntry(lenf,
(scan - lenb) - (lastscan + lenf),
(pos - lenb) - (lastpos + lenf))))
@@ -240,7 +242,7 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
lastoffset=pos-scan;
};
};
if (!patch->Close())
if (!diff_encoder.Close())
errx(1, "Closing the patch file");
if (I_cache == nullptr)
+4 -2
View File
@@ -53,8 +53,9 @@
},
'sources': [
'bsdiff.cc',
"bz2_compressor.cc",
"patch_writer.cc",
'bz2_compressor.cc',
'diff_encoder.cc',
'patch_writer.cc',
],
},
# bsdiff executable
@@ -126,6 +127,7 @@
'sources': [
'bsdiff_unittest.cc',
'bspatch_unittest.cc',
'diff_encoder_unittest.cc',
'extents_file_unittest.cc',
'extents_unittest.cc',
'patch_writer_unittest.cc',
+83
View File
@@ -0,0 +1,83 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "bsdiff/diff_encoder.h"
#include <vector>
#include "bsdiff/logging.h"
using std::endl;
namespace {
// The maximum positive number that we should encode. A number larger than this
// for unsigned fields will be interpreted as a negative value and thus a
// corrupt patch.
const uint64_t kMaxEncodedUint64Value = (1ULL << 63) - 1;
} // namespace
namespace bsdiff {
bool DiffEncoder::AddControlEntry(const ControlEntry& entry) {
if (entry.diff_size > kMaxEncodedUint64Value) {
LOG(ERROR) << "Encoding value out of range " << entry.diff_size << endl;
return false;
}
if (entry.extra_size > kMaxEncodedUint64Value) {
LOG(ERROR) << "Encoding value out of range " << entry.extra_size << endl;
return false;
}
if (written_output_ + entry.diff_size + entry.extra_size > new_size_) {
LOG(ERROR) << "Wrote more output than the declared new_size" << endl;
return false;
}
if (entry.diff_size > 0 &&
(old_pos_ < 0 || static_cast<uint64_t>(old_pos_) >= old_size_)) {
LOG(ERROR) << "The pointer in the old stream (" << old_pos_
<< ") is out of bounds [0, " << old_size_ << ")" << endl;
return false;
}
// Pass down the control entry.
if (!patch_->AddControlEntry(entry))
return false;
// Generate the diff stream.
std::vector<uint8_t> diff(entry.diff_size);
for (uint64_t i = 0; i < entry.diff_size; ++i) {
diff[i] = new_buf_[written_output_ + i] - old_buf_[old_pos_ + i];
}
if (!patch_->WriteDiffStream(diff.data(), diff.size())) {
LOG(ERROR) << "Writing " << diff.size() << " bytes to the diff stream"
<< endl;
return false;
}
if (!patch_->WriteExtraStream(new_buf_ + written_output_ + entry.diff_size,
entry.extra_size)) {
LOG(ERROR) << "Writing " << entry.extra_size << " bytes to the extra stream"
<< endl;
return false;
}
old_pos_ += entry.diff_size + entry.offset_increment;
written_output_ += entry.diff_size + entry.extra_size;
return true;
}
bool DiffEncoder::Close() {
if (written_output_ != new_size_) {
LOG(ERROR) << "Close() called but not all the output was written" << endl;
return false;
}
return patch_->Close();
}
} // namespace bsdiff
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef _BSDIFF_DIFF_ENCODER_H_
#define _BSDIFF_DIFF_ENCODER_H_
#include <stdint.h>
#include "bsdiff/bz2_compressor.h"
#include "bsdiff/patch_writer_interface.h"
namespace bsdiff {
// Helper class to encapsulate the diff and extra stream generation logic
// derived from the old and new file buffers. Using this class is impossible to
// produce an invalid or incomplete bsdiff patch, since it has checks in place
// verifying its correct usage.
class DiffEncoder {
public:
// Initialize the DiffEncoder with the old and new file buffers, as well as
// the path writer used. The |patch| must be already initialized.
DiffEncoder(PatchWriterInterface* patch,
const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size)
: patch_(patch),
old_buf_(old_buf),
old_size_(old_size),
new_buf_(new_buf),
new_size_(new_size) {}
// Add a new control triplet entry to the patch. The |entry.diff_size| bytes
// for the diff stream and the |entry.extra_size| bytes for the extra stream
// will be computed and added to the corresponding streams in the patch.
// Returns whether the operation succeeded. The operation can fail if either
// the old or new files are referenced out of bounds.
bool AddControlEntry(const ControlEntry& entry);
// Finalize the patch writing process and close the underlying patch writer.
bool Close();
private:
// Pointer to the patch we are writing to.
PatchWriterInterface* patch_;
// Old and new file buffers.
const uint8_t* old_buf_;
uint64_t old_size_;
const uint8_t* new_buf_;
uint64_t new_size_;
// Bytes of the new_buf_ already written.
uint64_t written_output_{0};
// The current position in the old buf.
int64_t old_pos_{0};
};
} // namespace bsdiff
#endif // _BSDIFF_DIFF_ENCODER_H_
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "bsdiff/diff_encoder.h"
#include <vector>
#include <gtest/gtest.h>
#include "bsdiff/fake_patch_writer.h"
#include "bsdiff/test_utils.h"
namespace {
// Generated with:
// echo 'Hello World' | hexdump -v -e '" " 12/1 "0x%02x, " "\n"'
const uint8_t kHelloWorld[] = {
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x0a,
};
} // namespace
namespace bsdiff {
class DiffEncoderTest : public testing::Test {
protected:
void SetUp() { EXPECT_TRUE(fake_patch_.Init()); }
FakePatchWriter fake_patch_;
std::unique_ptr<DiffEncoder> diff_encoder_;
};
TEST_F(DiffEncoderTest, CreateEmptyPatchTest) {
diff_encoder_.reset(new DiffEncoder(&fake_patch_, nullptr, 0, nullptr, 0));
EXPECT_TRUE(diff_encoder_->Close());
// Both diff and extra stream must be empty stream, and not control entries.
EXPECT_EQ(0U, fake_patch_.entries().size());
EXPECT_TRUE(fake_patch_.diff_stream().empty());
EXPECT_TRUE(fake_patch_.extra_stream().empty());
}
TEST_F(DiffEncoderTest, AllInExtraStreamTest) {
diff_encoder_.reset(new DiffEncoder(&fake_patch_, nullptr, 0, kHelloWorld,
sizeof(kHelloWorld)));
// Write to the extra stream in two parts: first 5 bytes, then the rest.
EXPECT_TRUE(diff_encoder_->AddControlEntry(ControlEntry(0, 5, 0)));
EXPECT_TRUE(diff_encoder_->AddControlEntry(
ControlEntry(0, sizeof(kHelloWorld) - 5, 0)));
EXPECT_TRUE(diff_encoder_->Close());
EXPECT_EQ(2U, fake_patch_.entries().size());
EXPECT_TRUE(fake_patch_.diff_stream().empty());
std::vector<uint8_t> hello_world(kHelloWorld,
kHelloWorld + sizeof(kHelloWorld));
EXPECT_EQ(hello_world, fake_patch_.extra_stream());
}
TEST_F(DiffEncoderTest, AllInDiffStreamTest) {
diff_encoder_.reset(new DiffEncoder(&fake_patch_, kHelloWorld,
sizeof(kHelloWorld), kHelloWorld,
sizeof(kHelloWorld)));
EXPECT_TRUE(
diff_encoder_->AddControlEntry(ControlEntry(sizeof(kHelloWorld), 0, 0)));
EXPECT_TRUE(diff_encoder_->Close());
EXPECT_EQ(std::vector<uint8_t>(sizeof(kHelloWorld), 0),
fake_patch_.diff_stream());
EXPECT_TRUE(fake_patch_.extra_stream().empty());
}
} // namespace bsdiff
+15 -12
View File
@@ -20,14 +20,9 @@ class FakePatchWriter : public PatchWriterInterface {
~FakePatchWriter() override = default;
// PatchWriterInterface overrides.
bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) override {
bool Init() override {
EXPECT_FALSE(initialized_);
initialized_ = true;
new_size_ = new_size;
old_size_ = old_size;
return true;
}
@@ -38,6 +33,16 @@ class FakePatchWriter : public PatchWriterInterface {
return true;
}
bool WriteDiffStream(const uint8_t* data, size_t size) override {
diff_stream_.insert(diff_stream_.end(), data, data + size);
return true;
}
bool WriteExtraStream(const uint8_t* data, size_t size) override {
extra_stream_.insert(extra_stream_.end(), data, data + size);
return true;
}
bool Close() override {
EXPECT_FALSE(closed_) << "Close() already called";
closed_ = true;
@@ -46,22 +51,20 @@ class FakePatchWriter : public PatchWriterInterface {
// Fake getter methods.
const std::vector<ControlEntry>& entries() const { return entries_; }
uint64_t new_size() const { return new_size_; }
uint64_t old_size() const { return old_size_; }
const std::vector<uint8_t>& diff_stream() const { return diff_stream_; }
const std::vector<uint8_t>& extra_stream() const { return extra_stream_; }
private:
// The list of ControlEntry passed to this class.
std::vector<ControlEntry> entries_;
std::vector<uint8_t> diff_stream_;
std::vector<uint8_t> extra_stream_;
// Whether this class was initialized.
bool initialized_{false};
// Whether the patch was closed.
bool closed_{false};
uint64_t new_size_;
uint64_t old_size_;
};
} // namespace bsdiff
+17 -14
View File
@@ -5,6 +5,10 @@
#ifndef _BSDIFF_PATCH_WRITER_INTERFACE_H_
#define _BSDIFF_PATCH_WRITER_INTERFACE_H_
#include <stdint.h>
#include <cstddef>
namespace bsdiff {
struct ControlEntry {
@@ -29,21 +33,20 @@ class PatchWriterInterface {
public:
virtual ~PatchWriterInterface() = default;
// Initialize the patch writer old and new files with the given buffers.
// These buffers are not copied into this class and must be valid until
// Close() is called. They will be used to encode the extra and diff streams
// when calling AddControlEntry(). Returns whether the initialization
// succeeded.
virtual bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) = 0;
// Initialize the patch writer.
virtual bool Init() = 0;
// Add a new control triplet entry to the patch. The |entry.diff_size| bytes
// for the diff stream and the |entry.extra_size| bytes for the extra stream
// will be computed and added to the corresponding streams in the patch.
// Returns whether the operation succeeded. The operation can fail if either
// the old or new files are referenced out of bounds.
// Write the passed |data| buffer of length |size| to the Diff or Extra
// streams respectively. Each method can be called independently from each
// other and calls don't need to be a correlation with the AddControlEntry()
// until Close() is called.
virtual bool WriteDiffStream(const uint8_t* data, size_t size) = 0;
virtual bool WriteExtraStream(const uint8_t* data, size_t size) = 0;
// Add a new control triplet entry to the patch. These triplets may be added
// at any point before calling Close(), regardless of whether the
// corresponding WriteDiffStream() and WriteExtraStream() have been called
// yet.
virtual bool AddControlEntry(const ControlEntry& entry) = 0;
// Finalize the patch writing process and close the file.
+13 -56
View File
@@ -12,11 +12,6 @@ using std::endl;
namespace {
// The maximum positive number that we should encode. A number larger than this
// for unsigned fields will be interpreted as a negative value and thus a
// corrupt patch.
const uint64_t kMaxEncodedUint64Value = (1ULL << 63) - 1;
constexpr uint8_t kMagicHeader[] = "BSDIFF40";
void EncodeInt64(int64_t x, uint8_t* buf) {
@@ -31,15 +26,7 @@ void EncodeInt64(int64_t x, uint8_t* buf) {
namespace bsdiff {
bool BsdiffPatchWriter::InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) {
old_buf_ = old_buf;
old_size_ = old_size;
new_buf_ = new_buf;
new_size_ = new_size;
bool BsdiffPatchWriter::Init() {
fp_ = fopen(patch_filename_.c_str(), "w");
if (!fp_) {
LOG(ERROR) << "Opening " << patch_filename_ << endl;
@@ -48,29 +35,15 @@ bool BsdiffPatchWriter::InitializeBuffers(const uint8_t* old_buf,
return true;
}
bool BsdiffPatchWriter::WriteDiffStream(const uint8_t* data, size_t size) {
return diff_stream_.Write(data, size);
}
bool BsdiffPatchWriter::WriteExtraStream(const uint8_t* data, size_t size) {
return extra_stream_.Write(data, size);
}
bool BsdiffPatchWriter::AddControlEntry(const ControlEntry& entry) {
if (entry.diff_size > kMaxEncodedUint64Value) {
LOG(ERROR) << "Encoding value out of range " << entry.diff_size << endl;
return false;
}
if (entry.extra_size > kMaxEncodedUint64Value) {
LOG(ERROR) << "Encoding value out of range " << entry.extra_size << endl;
return false;
}
if (written_output_ + entry.diff_size + entry.extra_size > new_size_) {
LOG(ERROR) << "Wrote more output than the declared new_size" << endl;
return false;
}
if (entry.diff_size > 0 &&
(old_pos_ < 0 || static_cast<uint64_t>(old_pos_) >= old_size_)) {
LOG(ERROR) << "The pointer in the old stream (" << old_pos_
<< ") is out of bounds [0, " << old_size_ << ")" << endl;
return false;
}
// Generate the 24 byte control entry.
uint8_t buf[24];
EncodeInt64(entry.diff_size, buf);
@@ -78,20 +51,6 @@ bool BsdiffPatchWriter::AddControlEntry(const ControlEntry& entry) {
EncodeInt64(entry.offset_increment, buf + 16);
if (!ctrl_stream_.Write(buf, sizeof(buf)))
return false;
// Generate the diff stream.
std::vector<uint8_t> diff(entry.diff_size);
for (uint64_t i = 0; i < entry.diff_size; ++i) {
diff[i] = new_buf_[written_output_ + i] - old_buf_[old_pos_ + i];
}
if (!diff_stream_.Write(diff.data(), diff.size()))
return false;
if (!extra_stream_.Write(new_buf_ + written_output_ + entry.diff_size,
entry.extra_size))
return false;
old_pos_ += entry.diff_size + entry.offset_increment;
written_output_ += entry.diff_size + entry.extra_size;
return true;
}
@@ -101,14 +60,12 @@ bool BsdiffPatchWriter::Close() {
LOG(ERROR) << "File not open." << endl;
return false;
}
if (written_output_ != new_size_) {
LOG(ERROR) << "Close() called but not all the output was written" << endl;
return false;
}
if (!ctrl_stream_.Finish() || !diff_stream_.Finish() ||
!extra_stream_.Finish())
!extra_stream_.Finish()) {
LOG(ERROR) << "Finalizing compressed streams." << endl;
return false;
}
auto ctrl_data = ctrl_stream_.GetCompressedData();
auto diff_data = diff_stream_.GetCompressedData();
@@ -155,7 +112,7 @@ bool BsdiffPatchWriter::WriteHeader(uint64_t ctrl_size, uint64_t diff_size) {
memcpy(header, kMagicHeader, 8);
EncodeInt64(ctrl_size, header + 8);
EncodeInt64(diff_size, header + 16);
EncodeInt64(new_size_, header + 24);
EncodeInt64(written_output_, header + 24);
if (fwrite(header, sizeof(header), 1, fp_) != 1) {
LOG(ERROR) << "writing to the patch file" << endl;
return false;
+5 -14
View File
@@ -23,10 +23,9 @@ class BsdiffPatchWriter : public PatchWriterInterface {
: patch_filename_(patch_filename) {}
// PatchWriterInterface overrides.
bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) override;
bool Init() override;
bool WriteDiffStream(const uint8_t* data, size_t size) override;
bool WriteExtraStream(const uint8_t* data, size_t size) override;
bool AddControlEntry(const ControlEntry& entry) override;
bool Close() override;
@@ -35,18 +34,10 @@ class BsdiffPatchWriter : public PatchWriterInterface {
// control block and the compressed diff block.
bool WriteHeader(uint64_t ctrl_size, uint64_t diff_size);
// Old and new file buffers.
const uint8_t* old_buf_{nullptr};
uint64_t old_size_{0};
const uint8_t* new_buf_{nullptr};
uint64_t new_size_{0};
// Bytes of the new_buf_ already written.
// Bytes of the new files already written. Needed to store the new length in
// the header of the file.
uint64_t written_output_{0};
// The current position in the old buf.
int64_t old_pos_{0};
// The current file we are writing to.
FILE* fp_{nullptr};
std::string patch_filename_;
+5 -5
View File
@@ -46,12 +46,13 @@ namespace bsdiff {
class BsdiffPatchWriterTest : public testing::Test {
protected:
void SetUp() override { EXPECT_TRUE(patch_writer_.Init()); }
test_utils::ScopedTempFile patch_file_{"bsdiff_newfile.XXXXXX"};
BsdiffPatchWriter patch_writer_{patch_file_.filename()};
};
TEST_F(BsdiffPatchWriterTest, CreateEmptyPatchTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(nullptr, 0, nullptr, 0));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;
@@ -64,12 +65,11 @@ TEST_F(BsdiffPatchWriterTest, CreateEmptyPatchTest) {
}
TEST_F(BsdiffPatchWriterTest, AllInExtraStreamTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(
nullptr, 0, kHelloWorld, sizeof(kHelloWorld)));
// Write to the extra stream in two parts: first 5 bytes, then the rest.
EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(0, 5, 0)));
EXPECT_TRUE(patch_writer_.AddControlEntry(
ControlEntry(0, sizeof(kHelloWorld) - 5, 0)));
EXPECT_TRUE(patch_writer_.WriteExtraStream(kHelloWorld, sizeof(kHelloWorld)));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;
@@ -87,11 +87,11 @@ TEST_F(BsdiffPatchWriterTest, AllInExtraStreamTest) {
}
TEST_F(BsdiffPatchWriterTest, AllInDiffStreamTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(
kHelloWorld, sizeof(kHelloWorld), kHelloWorld, sizeof(kHelloWorld)));
// Write to the extra stream in two parts: first 5 bytes, then the rest.
EXPECT_TRUE(
patch_writer_.AddControlEntry(ControlEntry(sizeof(kHelloWorld), 0, 0)));
std::vector<uint8_t> zeros(sizeof(kHelloWorld), 0);
EXPECT_TRUE(patch_writer_.WriteDiffStream(zeros.data(), zeros.size()));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;