Merge changes I9350157e,Ie450b279,Icc85d8b4

* changes:
  Add unittest for the PatchWriter.
  Make BsdiffPatchWriter into an Interface.
  Update .gitignore to exclude tracked files.
This commit is contained in:
Treehugger Robot authored and Gerrit Code Review committed 2017-09-29 11:34:47 +00:00
commit bfef6ef102
12 files changed
+369 -98

No files matched your search

+5 -4
View File
@@ -4,11 +4,12 @@
# any tracked files which get ignored after the change.
#
# Standard ignores
# Ignore object files and backup files
*.o
*.so
*~
# Executables
bsdiff
bspatch
unittests
/bsdiff
/bspatch
/bsdiff_unittest
+6 -1
View File
@@ -101,10 +101,10 @@ cc_test {
"bspatch_unittest.cc",
"extents_file_unittest.cc",
"extents_unittest.cc",
"patch_writer_unittest.cc",
"test_utils.cc",
"testrunner.cc",
],
cflags: ["-DBSDIFF_TARGET_UNITTEST"],
static_libs: [
"libbsdiff",
"libbspatch",
@@ -113,4 +113,9 @@ cc_test {
"libdivsufsort",
"libbz",
],
target: {
android: {
cflags: ["-DBSDIFF_TARGET_UNITTEST"],
},
},
}
+1
View File
@@ -53,6 +53,7 @@ bsdiff_common_unittests := \
bspatch_unittest.cc \
extents_file_unittest.cc \
extents_unittest.cc \
patch_writer_unittest.cc \
test_utils.cc \
testrunner.cc
+18 -12
View File
@@ -118,12 +118,17 @@ int bsdiff(const char* old_filename, const char* new_filename,
return ret;
}
// Generate bsdiff patch from |old_buf| to |new_buf|, save the patch file to
// |patch_filename|. Returns 0 on success.
// |I_cache| can be used to cache the suffix array if the same |old_buf| is used
// repeatedly, pass nullptr if not needed.
// TODO(deymo): Deprecate this version of the interface and move all callers
// to the underlying version using PatchWriterInterface instead. This allows
// more flexible options including different encodings.
int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
off_t newsize, const char* patch_filename, saidx_t** I_cache) {
BsdiffPatchWriter patch(patch_filename);
return bsdiff(old_buf, oldsize, new_buf, newsize, &patch, I_cache);
}
int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
off_t newsize, PatchWriterInterface* patch, saidx_t** I_cache) {
saidx_t *I;
off_t scan,pos=0,len;
off_t lastscan,lastpos,lastoffset;
@@ -131,7 +136,6 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
off_t s,Sf,lenf,Sb,lenb;
off_t overlap,Ss,lens;
off_t i;
BsdiffPatchWriter patch(old_buf, oldsize, new_buf, newsize);
if (I_cache && *I_cache) {
I = *I_cache;
@@ -139,13 +143,14 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
if ((I=static_cast<saidx_t*>(malloc((oldsize+1)*sizeof(saidx_t))))==NULL)
err(1,NULL);
if (divsufsort(old_buf, I, oldsize)) err(1, "divsufsort");
// Note: divsufsort() fails when the passed size is 0 and old_buf is NULL.
if (oldsize > 0 && divsufsort(old_buf, I, oldsize)) err(1, "divsufsort");
if (I_cache)
*I_cache = I;
}
/* Create the patch file */
if (!patch.Open(patch_filename))
/* Initialize the patch file */
if (!patch->InitializeBuffers(old_buf, oldsize, new_buf, newsize))
return 1;
/* Compute the differences, writing ctrl as we go */
@@ -224,9 +229,10 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
lenb-=lens;
};
if (!patch.AddControlEntry(ControlEntry(lenf,
(scan - lenb) - (lastscan + lenf),
(pos - lenb) - (lastpos + lenf))))
if (!patch->AddControlEntry(
ControlEntry(lenf,
(scan - lenb) - (lastscan + lenf),
(pos - lenb) - (lastpos + lenf))))
errx(1, "Writing a control entry");
lastscan=scan-lenb;
@@ -234,7 +240,7 @@ int bsdiff(const u_char* old_buf, off_t oldsize, const u_char* new_buf,
lastoffset=pos-scan;
};
};
if (!patch.Close())
if (!patch->Close())
errx(1, "Closing the patch file");
if (I_cache == nullptr)
+1
View File
@@ -128,6 +128,7 @@
'bspatch_unittest.cc',
'extents_file_unittest.cc',
'extents_unittest.cc',
'patch_writer_unittest.cc',
'test_utils.cc',
],
},
+53 -31
View File
@@ -5,58 +5,80 @@
#include "bsdiff/bsdiff.h"
#include <gtest/gtest.h>
#include <algorithm>
#include <random>
#include <string>
#include <vector>
#include "bsdiff/test_utils.h"
#include "bsdiff/fake_patch_writer.h"
using test_utils::BsdiffPatchFile;
namespace {
// Generate deterministic random data in the output buffer. The buffer must be
// already allocated with the desired size. The data generated depends on the
// selected size.
void GenerateRandomBuffer(std::vector<uint8_t>* buffer) {
std::minstd_rand prng(1234 + buffer->size());
std::generate(buffer->begin(), buffer->end(), prng);
}
} // namespace
namespace bsdiff {
class BsdiffTest : public testing::Test {
protected:
BsdiffTest()
: old_file_("bsdiff_oldfile.XXXXXX"),
new_file_("bsdiff_newfile.XXXXXX"),
patch_file_("bsdiff_patchfile.XXXXXX") {
}
~BsdiffTest() override {}
BsdiffTest() = default;
~BsdiffTest() override = default;
test_utils::ScopedTempFile old_file_;
test_utils::ScopedTempFile new_file_;
test_utils::ScopedTempFile patch_file_;
void RunBsdiff() {
EXPECT_EQ(0, bsdiff(old_file_.data(), old_file_.size(), new_file_.data(),
new_file_.size(), &patch_writer_, nullptr));
}
std::vector<uint8_t> old_file_;
std::vector<uint8_t> new_file_;
FakePatchWriter patch_writer_;
};
// Check that a file with no changes has a very small patch (no extra data).
TEST_F(BsdiffTest, EqualEmptyFiles) {
// Empty old and new files.
EXPECT_EQ(0, bsdiff(old_file_.filename().c_str(),
new_file_.filename().c_str(),
patch_file_.filename().c_str()));
BsdiffPatchFile patch;
EXPECT_TRUE(patch.LoadFromFile(patch_file_.filename()));
EXPECT_TRUE(patch.IsValid());
RunBsdiff();
// An empty bz2 file will have 14 bytes.
EXPECT_EQ(14, patch.diff_len);
EXPECT_EQ(14U, patch.extra_len);
// No entries should be generated on an empty new file.
EXPECT_TRUE(patch_writer_.entries().empty());
}
TEST_F(BsdiffTest, EqualSmallFiles) {
std::string some_text = "Hello world!";
std::vector<uint8_t> vec_some_text(some_text.begin(), some_text.end());
test_utils::WriteFile(old_file_.filename(), vec_some_text);
EXPECT_EQ(0, bsdiff(old_file_.filename().c_str(),
new_file_.filename().c_str(),
patch_file_.filename().c_str()));
BsdiffPatchFile patch;
EXPECT_TRUE(patch.LoadFromFile(patch_file_.filename()));
EXPECT_TRUE(patch.IsValid());
old_file_.insert(old_file_.begin(), some_text.begin(), some_text.end());
new_file_.insert(new_file_.begin(), some_text.begin(), some_text.end());
RunBsdiff();
// An empty bz2 file will have 14 bytes.
EXPECT_EQ(14, patch.diff_len);
EXPECT_EQ(14U, patch.extra_len);
EXPECT_EQ(1U, patch_writer_.entries().size());
ControlEntry entry = patch_writer_.entries()[0];
EXPECT_EQ(some_text.size(), entry.diff_size);
EXPECT_EQ(0U, entry.extra_size);
}
TEST_F(BsdiffTest, FileWithSmallErrorsTest) {
old_file_.resize(100);
GenerateRandomBuffer(&old_file_);
new_file_ = old_file_;
// Break a few bytes somewhere in the middle.
new_file_[20]++;
new_file_[30] += 2;
new_file_[31] += 2;
RunBsdiff();
// We expect that the result has only one entry with all in the diff stream
// since the two files are very similar.
EXPECT_EQ(1U, patch_writer_.entries().size());
ControlEntry entry = patch_writer_.entries()[0];
EXPECT_EQ(100U, entry.diff_size);
EXPECT_EQ(0U, entry.extra_size);
}
} // namespace bsdiff
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef _BSDIFF_FAKE_PATCH_WRITER_H_
#define _BSDIFF_FAKE_PATCH_WRITER_H_
#include <gtest/gtest.h>
#include <vector>
#include "bsdiff/patch_writer_interface.h"
namespace bsdiff {
// A fake PatchWriterInterface derived class with easy access to the data passed
// to it.
class FakePatchWriter : public PatchWriterInterface {
public:
FakePatchWriter() = default;
~FakePatchWriter() override = default;
// PatchWriterInterface overrides.
bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) override {
EXPECT_FALSE(initialized_);
initialized_ = true;
new_size_ = new_size;
old_size_ = old_size;
return true;
}
bool AddControlEntry(const ControlEntry& entry) override {
EXPECT_TRUE(initialized_);
EXPECT_FALSE(closed_);
entries_.push_back(entry);
return true;
}
bool Close() override {
EXPECT_FALSE(closed_) << "Close() already called";
closed_ = true;
return true;
}
// Fake getter methods.
const std::vector<ControlEntry>& entries() const { return entries_; }
uint64_t new_size() const { return new_size_; }
uint64_t old_size() const { return old_size_; }
private:
// The list of ControlEntry passed to this class.
std::vector<ControlEntry> entries_;
// Whether this class was initialized.
bool initialized_{false};
// Whether the patch was closed.
bool closed_{false};
uint64_t new_size_;
uint64_t old_size_;
};
} // namespace bsdiff
#endif // _BSDIFF_FAKE_PATCH_WRITER_H_
+15
View File
@@ -16,14 +16,21 @@
#endif
#include "bsdiff/common.h"
#include "bsdiff/patch_writer_interface.h"
namespace bsdiff {
// Generate bsdiff patch from the |old_filename| file to the |new_filename|
// file storing the resulting patch in a new |patch_filename| file.
BSDIFF_EXPORT
int bsdiff(const char* old_filename,
const char* new_filename,
const char* patch_filename);
// Generate bsdiff patch from |old_buf| to |new_buf|, save the patch file to
// |patch_filename|. Returns 0 on success.
// |I_cache| can be used to cache the suffix array if the same |old_buf| is used
// repeatedly, pass nullptr if not needed.
BSDIFF_EXPORT
int bsdiff(const u_char* old_buf,
off_t oldsize,
@@ -32,6 +39,14 @@ int bsdiff(const u_char* old_buf,
const char* patch_filename,
saidx_t** I_cache);
BSDIFF_EXPORT
int bsdiff(const u_char* old_buf,
off_t oldsize,
const u_char* new_buf,
off_t newsize,
PatchWriterInterface* patch,
saidx_t** I_cache);
} // namespace bsdiff
#endif // _BSDIFF_BSDIFF_H_
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef _BSDIFF_PATCH_WRITER_INTERFACE_H_
#define _BSDIFF_PATCH_WRITER_INTERFACE_H_
namespace bsdiff {
struct ControlEntry {
ControlEntry(uint64_t diff_size,
uint64_t extra_size,
int64_t offset_increment)
: diff_size(diff_size),
extra_size(extra_size),
offset_increment(offset_increment) {}
// The number of bytes to copy from the source and diff stream.
uint64_t diff_size;
// The number of bytes to copy from the extra stream.
uint64_t extra_size;
// The value to add to the source pointer after patching from the diff stream.
int64_t offset_increment;
};
class PatchWriterInterface {
public:
virtual ~PatchWriterInterface() = default;
// Initialize the patch writer old and new files with the given buffers.
// These buffers are not copied into this class and must be valid until
// Close() is called. They will be used to encode the extra and diff streams
// when calling AddControlEntry(). Returns whether the initialization
// succeeded.
virtual bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) = 0;
// Add a new control triplet entry to the patch. The |entry.diff_size| bytes
// for the diff stream and the |entry.extra_size| bytes for the extra stream
// will be computed and added to the corresponding streams in the patch.
// Returns whether the operation succeeded. The operation can fail if either
// the old or new files are referenced out of bounds.
virtual bool AddControlEntry(const ControlEntry& entry) = 0;
// Finalize the patch writing process and close the file.
virtual bool Close() = 0;
protected:
PatchWriterInterface() = default;
};
} // namespace bsdiff
#endif // _BSDIFF_PATCH_WRITER_INTERFACE_H_
+11 -3
View File
@@ -31,10 +31,18 @@ void EncodeInt64(int64_t x, uint8_t* buf) {
namespace bsdiff {
bool BsdiffPatchWriter::Open(const std::string& patch_filename) {
fp_ = fopen(patch_filename.c_str(), "w");
bool BsdiffPatchWriter::InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) {
old_buf_ = old_buf;
old_size_ = old_size;
new_buf_ = new_buf;
new_size_ = new_size;
fp_ = fopen(patch_filename_.c_str(), "w");
if (!fp_) {
LOG(ERROR) << "Opening " << patch_filename << endl;
LOG(ERROR) << "Opening " << patch_filename_ << endl;
return false;
}
return true;
+22 -47
View File
@@ -5,67 +5,41 @@
#ifndef _BSDIFF_PATCH_WRITER_H_
#define _BSDIFF_PATCH_WRITER_H_
#include <stdio.h>
#include <string>
#include <vector>
#include "bsdiff/common.h"
#include "bz2_compressor.h"
#include "bsdiff/bz2_compressor.h"
#include "bsdiff/patch_writer_interface.h"
namespace bsdiff {
struct ControlEntry {
ControlEntry(uint64_t diff_size,
uint64_t extra_size,
int64_t offset_increment)
: diff_size(diff_size),
extra_size(extra_size),
offset_increment(offset_increment) {}
// The number of bytes to copy from the source and diff stream.
uint64_t diff_size;
// The number of bytes to copy from the extra stream.
uint64_t extra_size;
// The value to add to the source pointer after patching from the diff stream.
int64_t offset_increment;
};
class BsdiffPatchWriter {
// A PatchWriterInterface class using the upstream's BSDIFF40 format: three
// BZ2-compressors and a 32-byte header.
class BsdiffPatchWriter : public PatchWriterInterface {
public:
BsdiffPatchWriter(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size)
: old_buf_(old_buf),
old_size_(old_size),
new_buf_(new_buf),
new_size_(new_size) {}
// Create the patch writer using the file |patch_filename| where the patch
// data will be written to.
explicit BsdiffPatchWriter(const std::string& patch_filename)
: patch_filename_(patch_filename) {}
// Create the file |patch_filename| where the patch will be written to.
bool Open(const std::string& patch_filename);
// Add a new control triplet entry to the patch. The |entry.diff_size| bytes
// for the diff stream and the |entry.extra_size| bytes for the extra stream
// will be computed and added to the corresponding streams in the patch.
// Returns whether the operation succeeded. The operation can fail if either
// the old or new files are referenced out of bounds.
bool AddControlEntry(const ControlEntry& entry);
bool Close();
// PatchWriterInterface overrides.
bool InitializeBuffers(const uint8_t* old_buf,
uint64_t old_size,
const uint8_t* new_buf,
uint64_t new_size) override;
bool AddControlEntry(const ControlEntry& entry) override;
bool Close() override;
private:
// Write the BSDIFF patch header to the |fp_| given the size of the compressed
// control block and the compressed diff block.
bool WriteHeader(uint64_t ctrl_size, uint64_t diff_size);
const uint8_t* old_buf_;
uint64_t old_size_;
const uint8_t* new_buf_;
uint64_t new_size_;
// Old and new file buffers.
const uint8_t* old_buf_{nullptr};
uint64_t old_size_{0};
const uint8_t* new_buf_{nullptr};
uint64_t new_size_{0};
// Bytes of the new_buf_ already written.
uint64_t written_output_{0};
@@ -75,6 +49,7 @@ class BsdiffPatchWriter {
// The current file we are writing to.
FILE* fp_{nullptr};
std::string patch_filename_;
// The three internal compressed streams.
BZ2Compressor ctrl_stream_;
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "bsdiff/patch_writer.h"
#include <gtest/gtest.h>
#include "bsdiff/test_utils.h"
namespace {
// Generated with:
// echo 'Hello World' | hexdump -v -e '" " 12/1 "0x%02x, " "\n"'
const uint8_t kHelloWorld[] = {
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x0a,
};
// Compressed empty file.
// bzip2 -9 </dev/null | hexdump -v -e '" " 7/1 "0x%02x, " "\n"'
const uint8_t kCompressedEmpty[] = {0x42, 0x5a, 0x68, 0x39, 0x17, 0x72, 0x45,
0x38, 0x50, 0x90, 0x00, 0x00, 0x00, 0x00};
// echo 'Hello World' | bzip2 -9 | hexdump -v -e '" " 12/1 "0x%02x, " "\n"'
const uint8_t kCompressedHelloWorld[] = {
0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xd8, 0x72,
0x01, 0x2f, 0x00, 0x00, 0x01, 0x57, 0x80, 0x00, 0x10, 0x40, 0x00, 0x00,
0x40, 0x00, 0x80, 0x06, 0x04, 0x90, 0x00, 0x20, 0x00, 0x22, 0x06, 0x86,
0xd4, 0x20, 0xc9, 0x88, 0xc7, 0x69, 0xe8, 0x28, 0x1f, 0x8b, 0xb9, 0x22,
0x9c, 0x28, 0x48, 0x6c, 0x39, 0x00, 0x97, 0x80,
};
// Compressed a buffer of zeros of the same length of the kHelloWorld.
// head -c `echo 'Hello World' | wc -c` /dev/zero | bzip2 -9 |
// hexdump -v -e '" " 10/1 "0x%02x, " "\n"'
const uint8_t kCompressedZeros[] = {
0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59,
0xf6, 0x63, 0xab, 0xde, 0x00, 0x00, 0x00, 0x40, 0x00, 0x40,
0x40, 0x20, 0x00, 0x21, 0x00, 0x82, 0x83, 0x17, 0x72, 0x45,
0x38, 0x50, 0x90, 0xf6, 0x63, 0xab, 0xde,
};
} // namespace
namespace bsdiff {
class BsdiffPatchWriterTest : public testing::Test {
protected:
test_utils::ScopedTempFile patch_file_{"bsdiff_newfile.XXXXXX"};
BsdiffPatchWriter patch_writer_{patch_file_.filename()};
};
TEST_F(BsdiffPatchWriterTest, CreateEmptyPatchTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(nullptr, 0, nullptr, 0));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;
EXPECT_TRUE(patch.LoadFromFile(patch_file_.filename()));
EXPECT_TRUE(patch.IsValid());
// An empty bz2 file will have 14 bytes.
EXPECT_EQ(sizeof(kCompressedEmpty), static_cast<uint64_t>(patch.diff_len));
EXPECT_EQ(sizeof(kCompressedEmpty), patch.extra_len);
}
TEST_F(BsdiffPatchWriterTest, AllInExtraStreamTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(
nullptr, 0, kHelloWorld, sizeof(kHelloWorld)));
// Write to the extra stream in two parts: first 5 bytes, then the rest.
EXPECT_TRUE(patch_writer_.AddControlEntry(ControlEntry(0, 5, 0)));
EXPECT_TRUE(patch_writer_.AddControlEntry(
ControlEntry(0, sizeof(kHelloWorld) - 5, 0)));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;
EXPECT_TRUE(patch.LoadFromFile(patch_file_.filename()));
EXPECT_TRUE(patch.IsValid());
EXPECT_EQ(patch.bz2_diff,
std::vector<uint8_t>(kCompressedEmpty,
kCompressedEmpty + sizeof(kCompressedEmpty)));
EXPECT_EQ(patch.bz2_extra,
std::vector<uint8_t>(
kCompressedHelloWorld,
kCompressedHelloWorld + sizeof(kCompressedHelloWorld)));
EXPECT_EQ(static_cast<int64_t>(sizeof(kHelloWorld)), patch.new_file_len);
}
TEST_F(BsdiffPatchWriterTest, AllInDiffStreamTest) {
EXPECT_TRUE(patch_writer_.InitializeBuffers(
kHelloWorld, sizeof(kHelloWorld), kHelloWorld, sizeof(kHelloWorld)));
// Write to the extra stream in two parts: first 5 bytes, then the rest.
EXPECT_TRUE(
patch_writer_.AddControlEntry(ControlEntry(sizeof(kHelloWorld), 0, 0)));
EXPECT_TRUE(patch_writer_.Close());
test_utils::BsdiffPatchFile patch;
EXPECT_TRUE(patch.LoadFromFile(patch_file_.filename()));
EXPECT_TRUE(patch.IsValid());
EXPECT_EQ(patch.bz2_extra,
std::vector<uint8_t>(kCompressedEmpty,
kCompressedEmpty + sizeof(kCompressedEmpty)));
EXPECT_EQ(patch.bz2_diff,
std::vector<uint8_t>(kCompressedZeros,
kCompressedZeros + sizeof(kCompressedZeros)));
EXPECT_EQ(static_cast<int64_t>(sizeof(kHelloWorld)), patch.new_file_len);
}
} // namespace bsdiff