[BinaryStream] Support growable streams.

The existing library assumed that a stream's length would never
change.  This makes some things simpler, but it's not flexible
enough for what we need, especially for writable streams where
what you really want is for each call to write to actually append.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@319070 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Zachary Turner 2017-11-27 18:48:37 +00:00
parent 5f26e59f29
commit 81a153c99a
10 changed files with 260 additions and 39 deletions

View File

@ -47,6 +47,11 @@ inline StringRef toStringRef(ArrayRef<uint8_t> Input) {
return StringRef(reinterpret_cast<const char *>(Input.begin()), Input.size());
}
/// Construct a string ref from an array ref of unsigned chars.
inline ArrayRef<uint8_t> arrayRefFromStringRef(StringRef Input) {
return {Input.bytes_begin(), Input.bytes_end()};
}
/// Interpret the given character \p C as a hexadecimal digit and return its
/// value.
///

View File

@ -41,7 +41,7 @@ public:
Error readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
Buffer = Data.slice(Offset, Size);
return Error::success();
@ -49,7 +49,7 @@ public:
Error readLongestContiguousChunk(uint32_t Offset,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, 1))
if (auto EC = checkOffsetForRead(Offset, 1))
return EC;
Buffer = Data.slice(Offset);
return Error::success();
@ -114,7 +114,7 @@ public:
if (Buffer.empty())
return Error::success();
if (auto EC = checkOffset(Offset, Buffer.size()))
if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
return EC;
uint8_t *DataPtr = const_cast<uint8_t *>(Data.data());
@ -131,6 +131,72 @@ private:
BinaryByteStream ImmutableStream;
};
/// \brief An implementation of WritableBinaryStream which can write at its end
/// causing the underlying data to grow. This class owns the underlying data.
class AppendingBinaryByteStream : public WritableBinaryStream {
std::vector<uint8_t> Data;
llvm::support::endianness Endian;
public:
AppendingBinaryByteStream() = default;
AppendingBinaryByteStream(llvm::support::endianness Endian)
: Endian(Endian) {}
void clear() { Data.clear(); }
llvm::support::endianness getEndian() const override { return Endian; }
Error readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
return EC;
Buffer = makeArrayRef(Data).slice(Offset, Size);
return Error::success();
}
Error readLongestContiguousChunk(uint32_t Offset,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffsetForWrite(Offset, 1))
return EC;
Buffer = makeArrayRef(Data).slice(Offset);
return Error::success();
}
uint32_t getLength() override { return Data.size(); }
Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> Buffer) override {
if (Buffer.empty())
return Error::success();
// This is well-defined for any case except where offset is strictly
// greater than the current length. If offset is equal to the current
// length, we can still grow. If offset is beyond the current length, we
// would have to decide how to deal with the intermediate uninitialized
// bytes. So we punt on that case for simplicity and just say it's an
// error.
if (Offset > getLength())
return make_error<BinaryStreamError>(stream_error_code::invalid_offset);
uint32_t RequiredSize = Offset + Buffer.size();
if (RequiredSize > Data.size())
Data.resize(RequiredSize);
::memcpy(Data.data() + Offset, Buffer.data(), Buffer.size());
return Error::success();
}
Error commit() override { return Error::success(); }
/// \brief Return the properties of this stream.
virtual BinaryStreamFlags getFlags() const override {
return BSF_Write | BSF_Append;
}
MutableArrayRef<uint8_t> data() { return Data; }
};
/// \brief An implementation of WritableBinaryStream backed by an llvm
/// FileOutputBuffer.
class FileBufferByteStream : public WritableBinaryStream {

View File

@ -45,7 +45,7 @@ public:
if (!ExpectedIndex)
return ExpectedIndex.takeError();
const auto &Item = Items[*ExpectedIndex];
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
if (Size > Traits::length(Item))
return make_error<BinaryStreamError>(stream_error_code::stream_too_short);

View File

@ -11,6 +11,7 @@
#define LLVM_SUPPORT_BINARYSTREAM_H
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/Support/BinaryStreamError.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
@ -18,6 +19,13 @@
namespace llvm {
enum BinaryStreamFlags {
BSF_None = 0,
BSF_Write = 1, // Stream supports writing.
BSF_Append = 2, // Writing can occur at offset == length.
LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ BSF_Append)
};
/// \brief An interface for accessing data in a stream-like format, but which
/// discourages copying. Instead of specifying a buffer in which to copy
/// data on a read, the API returns an ArrayRef to data owned by the stream's
@ -45,8 +53,11 @@ public:
/// \brief Return the number of bytes of data in this stream.
virtual uint32_t getLength() = 0;
/// \brief Return the properties of this stream.
virtual BinaryStreamFlags getFlags() const { return BSF_None; }
protected:
Error checkOffset(uint32_t Offset, uint32_t DataSize) {
Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) {
if (Offset > getLength())
return make_error<BinaryStreamError>(stream_error_code::invalid_offset);
if (getLength() < DataSize + Offset)
@ -71,6 +82,19 @@ public:
/// \brief For buffered streams, commits changes to the backing store.
virtual Error commit() = 0;
/// \brief Return the properties of this stream.
BinaryStreamFlags getFlags() const override { return BSF_Write; }
protected:
Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) {
if (!(getFlags() & BSF_Append))
return checkOffsetForRead(Offset, DataSize);
if (Offset > getLength())
return make_error<BinaryStreamError>(stream_error_code::invalid_offset);
return Error::success();
}
};
} // end namespace llvm

View File

@ -11,6 +11,7 @@
#define LLVM_SUPPORT_BINARYSTREAMREF_H
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/Optional.h"
#include "llvm/Support/BinaryStream.h"
#include "llvm/Support/BinaryStreamError.h"
#include "llvm/Support/Error.h"
@ -24,12 +25,18 @@ namespace llvm {
template <class RefType, class StreamType> class BinaryStreamRefBase {
protected:
BinaryStreamRefBase() = default;
explicit BinaryStreamRefBase(StreamType &BorrowedImpl)
: BorrowedImpl(&BorrowedImpl), ViewOffset(0) {
if (!(BorrowedImpl.getFlags() & BSF_Append))
Length = BorrowedImpl.getLength();
}
BinaryStreamRefBase(std::shared_ptr<StreamType> SharedImpl, uint32_t Offset,
uint32_t Length)
Optional<uint32_t> Length)
: SharedImpl(SharedImpl), BorrowedImpl(SharedImpl.get()),
ViewOffset(Offset), Length(Length) {}
BinaryStreamRefBase(StreamType &BorrowedImpl, uint32_t Offset,
uint32_t Length)
Optional<uint32_t> Length)
: BorrowedImpl(&BorrowedImpl), ViewOffset(Offset), Length(Length) {}
BinaryStreamRefBase(const BinaryStreamRefBase &Other) = default;
BinaryStreamRefBase &operator=(const BinaryStreamRefBase &Other) = default;
@ -42,28 +49,50 @@ public:
return BorrowedImpl->getEndian();
}
uint32_t getLength() const { return Length; }
uint32_t getLength() const {
if (Length.hasValue())
return *Length;
/// Return a new BinaryStreamRef with the first \p N elements removed.
return BorrowedImpl ? (BorrowedImpl->getLength() - ViewOffset) : 0;
}
/// Return a new BinaryStreamRef with the first \p N elements removed. If
/// this BinaryStreamRef is length-tracking, then the resulting one will be
/// too.
RefType drop_front(uint32_t N) const {
if (!BorrowedImpl)
return RefType();
N = std::min(N, Length);
N = std::min(N, getLength());
RefType Result(static_cast<const RefType &>(*this));
if (N == 0)
return Result;
Result.ViewOffset += N;
Result.Length -= N;
if (Result.Length.hasValue())
*Result.Length -= N;
return Result;
}
/// Return a new BinaryStreamRef with the first \p N elements removed.
/// Return a new BinaryStreamRef with the last \p N elements removed. If
/// this BinaryStreamRef is length-tracking and \p N is greater than 0, then
/// this BinaryStreamRef will no longer length-track.
RefType drop_back(uint32_t N) const {
if (!BorrowedImpl)
return RefType();
N = std::min(N, Length);
RefType Result(static_cast<const RefType &>(*this));
Result.Length -= N;
N = std::min(N, getLength());
if (N == 0)
return Result;
// Since we're dropping non-zero bytes from the end, stop length-tracking
// by setting the length of the resulting StreamRef to an explicit value.
if (!Result.Length.hasValue())
Result.Length = getLength();
*Result.Length -= N;
return Result;
}
@ -104,7 +133,7 @@ public:
}
protected:
Error checkOffset(uint32_t Offset, uint32_t DataSize) const {
Error checkOffsetForRead(uint32_t Offset, uint32_t DataSize) const {
if (Offset > getLength())
return make_error<BinaryStreamError>(stream_error_code::invalid_offset);
if (getLength() < DataSize + Offset)
@ -115,7 +144,7 @@ protected:
std::shared_ptr<StreamType> SharedImpl;
StreamType *BorrowedImpl = nullptr;
uint32_t ViewOffset = 0;
uint32_t Length = 0;
Optional<uint32_t> Length;
};
/// \brief BinaryStreamRef is to BinaryStream what ArrayRef is to an Array. It
@ -130,13 +159,14 @@ class BinaryStreamRef
friend BinaryStreamRefBase<BinaryStreamRef, BinaryStream>;
friend class WritableBinaryStreamRef;
BinaryStreamRef(std::shared_ptr<BinaryStream> Impl, uint32_t ViewOffset,
uint32_t Length)
Optional<uint32_t> Length)
: BinaryStreamRefBase(Impl, ViewOffset, Length) {}
public:
BinaryStreamRef() = default;
BinaryStreamRef(BinaryStream &Stream);
BinaryStreamRef(BinaryStream &Stream, uint32_t Offset, uint32_t Length);
BinaryStreamRef(BinaryStream &Stream, uint32_t Offset,
Optional<uint32_t> Length);
explicit BinaryStreamRef(ArrayRef<uint8_t> Data,
llvm::support::endianness Endian);
explicit BinaryStreamRef(StringRef Data, llvm::support::endianness Endian);
@ -195,14 +225,23 @@ class WritableBinaryStreamRef
WritableBinaryStream> {
friend BinaryStreamRefBase<WritableBinaryStreamRef, WritableBinaryStream>;
WritableBinaryStreamRef(std::shared_ptr<WritableBinaryStream> Impl,
uint32_t ViewOffset, uint32_t Length)
uint32_t ViewOffset, Optional<uint32_t> Length)
: BinaryStreamRefBase(Impl, ViewOffset, Length) {}
Error checkOffsetForWrite(uint32_t Offset, uint32_t DataSize) const {
if (!(BorrowedImpl->getFlags() & BSF_Append))
return checkOffsetForRead(Offset, DataSize);
if (Offset > getLength())
return make_error<BinaryStreamError>(stream_error_code::invalid_offset);
return Error::success();
}
public:
WritableBinaryStreamRef() = default;
WritableBinaryStreamRef(WritableBinaryStream &Stream);
WritableBinaryStreamRef(WritableBinaryStream &Stream, uint32_t Offset,
uint32_t Length);
Optional<uint32_t> Length);
explicit WritableBinaryStreamRef(MutableArrayRef<uint8_t> Data,
llvm::support::endianness Endian);
WritableBinaryStreamRef(const WritableBinaryStreamRef &Other) = default;

View File

@ -89,7 +89,7 @@ MappedBlockStream::createFpmStream(const MSFLayout &Layout,
Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) {
// Make sure we aren't trying to read beyond the end of the stream.
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
if (tryReadContiguously(Offset, Size, Buffer))
@ -167,7 +167,7 @@ Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
Error MappedBlockStream::readLongestContiguousChunk(uint32_t Offset,
ArrayRef<uint8_t> &Buffer) {
// Make sure we aren't trying to read beyond the end of the stream.
if (auto EC = checkOffset(Offset, 1))
if (auto EC = checkOffsetForRead(Offset, 1))
return EC;
uint32_t First = Offset / BlockSize;
@ -243,7 +243,7 @@ Error MappedBlockStream::readBytes(uint32_t Offset,
uint32_t OffsetInBlock = Offset % BlockSize;
// Make sure we aren't trying to read beyond the end of the stream.
if (auto EC = checkOffset(Offset, Buffer.size()))
if (auto EC = checkOffsetForRead(Offset, Buffer.size()))
return EC;
uint32_t BytesLeft = Buffer.size();
@ -388,7 +388,7 @@ uint32_t WritableMappedBlockStream::getLength() {
Error WritableMappedBlockStream::writeBytes(uint32_t Offset,
ArrayRef<uint8_t> Buffer) {
// Make sure we aren't trying to write beyond the end of the stream.
if (auto EC = checkOffset(Offset, Buffer.size()))
if (auto EC = checkOffsetForWrite(Offset, Buffer.size()))
return EC;
uint32_t BlockNum = Offset / getBlockSize();

View File

@ -66,9 +66,9 @@ private:
}
BinaryStreamRef::BinaryStreamRef(BinaryStream &Stream)
: BinaryStreamRef(Stream, 0, Stream.getLength()) {}
: BinaryStreamRefBase(Stream) {}
BinaryStreamRef::BinaryStreamRef(BinaryStream &Stream, uint32_t Offset,
uint32_t Length)
Optional<uint32_t> Length)
: BinaryStreamRefBase(Stream, Offset, Length) {}
BinaryStreamRef::BinaryStreamRef(ArrayRef<uint8_t> Data, endianness Endian)
: BinaryStreamRefBase(std::make_shared<ArrayRefImpl>(Data, Endian), 0,
@ -79,14 +79,14 @@ BinaryStreamRef::BinaryStreamRef(StringRef Data, endianness Endian)
Error BinaryStreamRef::readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) const {
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
return BorrowedImpl->readBytes(ViewOffset + Offset, Size, Buffer);
}
Error BinaryStreamRef::readLongestContiguousChunk(
uint32_t Offset, ArrayRef<uint8_t> &Buffer) const {
if (auto EC = checkOffset(Offset, 1))
if (auto EC = checkOffsetForRead(Offset, 1))
return EC;
if (auto EC =
@ -95,18 +95,18 @@ Error BinaryStreamRef::readLongestContiguousChunk(
// This StreamRef might refer to a smaller window over a larger stream. In
// that case we will have read out more bytes than we should return, because
// we should not read past the end of the current view.
uint32_t MaxLength = Length - Offset;
uint32_t MaxLength = getLength() - Offset;
if (Buffer.size() > MaxLength)
Buffer = Buffer.slice(0, MaxLength);
return Error::success();
}
WritableBinaryStreamRef::WritableBinaryStreamRef(WritableBinaryStream &Stream)
: WritableBinaryStreamRef(Stream, 0, Stream.getLength()) {}
: BinaryStreamRefBase(Stream) {}
WritableBinaryStreamRef::WritableBinaryStreamRef(WritableBinaryStream &Stream,
uint32_t Offset,
uint32_t Length)
Optional<uint32_t> Length)
: BinaryStreamRefBase(Stream, Offset, Length) {}
WritableBinaryStreamRef::WritableBinaryStreamRef(MutableArrayRef<uint8_t> Data,
@ -117,7 +117,7 @@ WritableBinaryStreamRef::WritableBinaryStreamRef(MutableArrayRef<uint8_t> Data,
Error WritableBinaryStreamRef::writeBytes(uint32_t Offset,
ArrayRef<uint8_t> Data) const {
if (auto EC = checkOffset(Offset, Data.size()))
if (auto EC = checkOffsetForWrite(Offset, Data.size()))
return EC;
return BorrowedImpl->writeBytes(ViewOffset + Offset, Data);

View File

@ -42,7 +42,8 @@ Error BinaryStreamWriter::writeCString(StringRef Str) {
}
Error BinaryStreamWriter::writeFixedString(StringRef Str) {
return writeBytes(ArrayRef<uint8_t>(Str.bytes_begin(), Str.bytes_end()));
return writeBytes(arrayRefFromStringRef(Str));
}
Error BinaryStreamWriter::writeStreamRef(BinaryStreamRef Ref) {

View File

@ -42,7 +42,7 @@ public:
Error readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
Buffer = Data.slice(Offset, Size);
return Error::success();
@ -50,7 +50,7 @@ public:
Error readLongestContiguousChunk(uint32_t Offset,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, 1))
if (auto EC = checkOffsetForRead(Offset, 1))
return EC;
Buffer = Data.drop_front(Offset);
return Error::success();
@ -59,7 +59,7 @@ public:
uint32_t getLength() override { return Data.size(); }
Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> SrcData) override {
if (auto EC = checkOffset(Offset, SrcData.size()))
if (auto EC = checkOffsetForWrite(Offset, SrcData.size()))
return EC;
::memcpy(&Data[Offset], SrcData.data(), SrcData.size());
return Error::success();

View File

@ -36,7 +36,7 @@ public:
Error readBytes(uint32_t Offset, uint32_t Size,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, Size))
if (auto EC = checkOffsetForRead(Offset, Size))
return EC;
uint32_t S = startIndex(Offset);
auto Ref = Data.drop_front(S);
@ -55,7 +55,7 @@ public:
Error readLongestContiguousChunk(uint32_t Offset,
ArrayRef<uint8_t> &Buffer) override {
if (auto EC = checkOffset(Offset, 1))
if (auto EC = checkOffsetForRead(Offset, 1))
return EC;
uint32_t S = startIndex(Offset);
Buffer = Data.drop_front(S);
@ -65,7 +65,7 @@ public:
uint32_t getLength() override { return Data.size(); }
Error writeBytes(uint32_t Offset, ArrayRef<uint8_t> SrcData) override {
if (auto EC = checkOffset(Offset, SrcData.size()))
if (auto EC = checkOffsetForWrite(Offset, SrcData.size()))
return EC;
if (SrcData.empty())
return Error::success();
@ -267,6 +267,56 @@ TEST_F(BinaryStreamTest, StreamRefBounds) {
}
}
TEST_F(BinaryStreamTest, StreamRefDynamicSize) {
StringRef Strings[] = {"1", "2", "3", "4"};
AppendingBinaryByteStream Stream(support::little);
BinaryStreamWriter Writer(Stream);
BinaryStreamReader Reader(Stream);
const uint8_t *Byte;
StringRef Str;
// When the stream is empty, it should report a 0 length and we should get an
// error trying to read even 1 byte from it.
BinaryStreamRef ConstRef(Stream);
EXPECT_EQ(0, ConstRef.getLength());
EXPECT_THAT_ERROR(Reader.readObject(Byte), Failed());
// But if we write to it, its size should increase and we should be able to
// read not just a byte, but the string that was written.
EXPECT_THAT_ERROR(Writer.writeCString(Strings[0]), Succeeded());
EXPECT_EQ(2, ConstRef.getLength());
EXPECT_THAT_ERROR(Reader.readObject(Byte), Succeeded());
Reader.setOffset(0);
EXPECT_THAT_ERROR(Reader.readCString(Str), Succeeded());
EXPECT_EQ(Str, Strings[0]);
// If we drop some bytes from the front, we should still track the length as
// the
// underlying stream grows.
BinaryStreamRef Dropped = ConstRef.drop_front(1);
EXPECT_EQ(1, Dropped.getLength());
EXPECT_THAT_ERROR(Writer.writeCString(Strings[1]), Succeeded());
EXPECT_EQ(4, ConstRef.getLength());
EXPECT_EQ(3, Dropped.getLength());
// If we drop zero bytes from the back, we should continue tracking the
// length.
Dropped = Dropped.drop_back(0);
EXPECT_THAT_ERROR(Writer.writeCString(Strings[2]), Succeeded());
EXPECT_EQ(6, ConstRef.getLength());
EXPECT_EQ(5, Dropped.getLength());
// If we drop non-zero bytes from the back, we should stop tracking the
// length.
Dropped = Dropped.drop_back(1);
EXPECT_THAT_ERROR(Writer.writeCString(Strings[3]), Succeeded());
EXPECT_EQ(8, ConstRef.getLength());
EXPECT_EQ(4, Dropped.getLength());
}
TEST_F(BinaryStreamTest, DropOperations) {
std::vector<uint8_t> InputData = {1, 2, 3, 4, 5, 4, 3, 2, 1};
auto RefData = makeArrayRef(InputData);
@ -345,6 +395,25 @@ TEST_F(BinaryStreamTest, MutableBinaryByteStreamBounds) {
}
}
TEST_F(BinaryStreamTest, AppendingStream) {
AppendingBinaryByteStream Stream(llvm::support::little);
EXPECT_EQ(0, Stream.getLength());
std::vector<uint8_t> InputData = {'T', 'e', 's', 't', 'T', 'e', 's', 't'};
auto Test = makeArrayRef(InputData).take_front(4);
// Writing past the end of the stream is an error.
EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Failed());
// Writing exactly at the end of the stream is ok.
EXPECT_THAT_ERROR(Stream.writeBytes(0, Test), Succeeded());
EXPECT_EQ(Test, Stream.data());
// And now that the end of the stream is where we couldn't write before, now
// we can write.
EXPECT_THAT_ERROR(Stream.writeBytes(4, Test), Succeeded());
EXPECT_EQ(MutableArrayRef<uint8_t>(InputData), Stream.data());
}
// Test that FixedStreamArray works correctly.
TEST_F(BinaryStreamTest, FixedStreamArray) {
std::vector<uint32_t> Ints = {90823, 12908, 109823, 209823};
@ -693,6 +762,23 @@ TEST_F(BinaryStreamTest, StringWriterStrings) {
EXPECT_EQ(makeArrayRef(Strings), makeArrayRef(InStrings));
}
}
TEST_F(BinaryStreamTest, StreamWriterAppend) {
StringRef Strings[] = {"First", "Second", "Third", "Fourth"};
AppendingBinaryByteStream Stream(support::little);
BinaryStreamWriter Writer(Stream);
for (auto &Str : Strings) {
EXPECT_THAT_ERROR(Writer.writeCString(Str), Succeeded());
}
BinaryStreamReader Reader(Stream);
for (auto &Str : Strings) {
StringRef S;
EXPECT_THAT_ERROR(Reader.readCString(S), Succeeded());
EXPECT_EQ(Str, S);
}
}
}
namespace {