Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-28 23:31:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1284031 (Part 1) - Advance SourceBufferIterator in Lex() per-state. r=njn

Browse Source
This commit is contained in:
Seth Fowler 2016-07-14 19:55:35 -07:00
parent 6fbb8890f0
commit 269c03d80e
2 changed files with 163 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

240
image/StreamingLexer.h
View File

@ -31,13 +31,16 @@ enum class BufferingStrategy
UNBUFFERED // Data will be processed as it arrives, in multiple chunks.
};
/// The result of a call to StreamingLexer::Lex().
/// Possible terminal states for the lexer.
enum class TerminalState
{
SUCCESS,
FAILURE
};
/// The result of a call to StreamingLexer::Lex().
typedef Variant<TerminalState> LexerResult;
/**
* LexerTransition is a type used to give commands to the lexing framework.
* Code that uses StreamingLexer can create LexerTransition values using the
@ -259,9 +262,11 @@ class StreamingLexer
{
public:
explicit StreamingLexer(LexerTransition<State> aStartState)
: mTransition(aStartState)
: mTransition(TerminalState::FAILURE)
, mToReadUnbuffered(0)
{ }
{
SetTransition(aStartState);
}
template <typename Func>
Maybe<TerminalState> Lex(SourceBufferIterator& aIterator,
@ -274,8 +279,15 @@ public:
return Some(mTransition.NextStateAsTerminal());
}
Maybe<LexerResult> result;
do {
switch (aIterator.AdvanceOrScheduleResume(SIZE_MAX, aOnResume)) {
// Figure out how much we need to read.
const size_t toRead = mTransition.Buffering() == BufferingStrategy::UNBUFFERED
? mToReadUnbuffered
: mTransition.Size() - mBuffer.length();
// Attempt to advance the iterator by |toRead| bytes.
switch (aIterator.AdvanceOrScheduleResume(toRead, aOnResume)) {
case SourceBufferIterator::WAITING:
// We can't continue because the rest of the data hasn't arrived from
// the network yet. We don't have to do anything special; the
@ -289,173 +301,123 @@ public:
// SourceBuffer was completed with a failing status, we want to fail.
// This happens only in exceptional situations like SourceBuffer
// itself encountering a failure due to OOM.
mTransition = NS_SUCCEEDED(aIterator.CompletionStatus())
? Transition::TerminateSuccess()
: Transition::TerminateFailure();
result = SetTransition(NS_SUCCEEDED(aIterator.CompletionStatus())
? Transition::TerminateSuccess()
: Transition::TerminateFailure());
break;
case SourceBufferIterator::READY:
// Process the new data that became available. This may result in us
// transitioning to a terminal state; we'll check if that happened at
// the bottom of the loop.
// Process the new data that became available.
MOZ_ASSERT(aIterator.Data());
MOZ_ASSERT(aIterator.Length() > 0);
Lex(aIterator.Data(), aIterator.Length(), aFunc);
result = mTransition.Buffering() == BufferingStrategy::UNBUFFERED
? UnbufferedRead(aIterator, aFunc)
: BufferedRead(aIterator, aFunc);
break;
default:
MOZ_ASSERT_UNREACHABLE("Unknown SourceBufferIterator state");
mTransition = Transition::TerminateFailure();
result = SetTransition(Transition::TerminateFailure());
}
} while (!mTransition.NextStateIsTerminal());
} while (!result);
// We're done. Return the terminal state.
return Some(mTransition.NextStateAsTerminal());
// Map |LexerResult| onto the old |Maybe<TerminalState>| API.
return result->is<TerminalState>() ? Some(result->as<TerminalState>())
: Nothing();
}
private:
template <typename Func>
Maybe<TerminalState> Lex(const char* aInput, size_t aLength, Func aFunc)
Maybe<LexerResult> UnbufferedRead(SourceBufferIterator& aIterator, Func aFunc)
{
MOZ_ASSERT(aInput);
if (mTransition.NextStateIsTerminal()) {
// We've already reached a terminal state. We never deliver any more data
// in this case; just return the terminal state again immediately.
return Some(mTransition.NextStateAsTerminal());
}
MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::UNBUFFERED);
MOZ_ASSERT(mBuffer.empty(),
"Buffered read at the same time as unbuffered read?");
if (mToReadUnbuffered > 0) {
// We're continuing an unbuffered read.
MOZ_ASSERT(mBuffer.empty(),
"Shouldn't be continuing an unbuffered read and a buffered "
"read at the same time");
size_t toRead = std::min(mToReadUnbuffered, aLength);
// Call aFunc with the unbuffered state to indicate that we're in the
// middle of an unbuffered read. We enforce that any state transition
// passed back to us is either a terminal state or takes us back to the
// unbuffered state.
LexerTransition<State> unbufferedTransition =
aFunc(mTransition.UnbufferedState(), aInput, toRead);
aFunc(mTransition.UnbufferedState(), aIterator.Data(), aIterator.Length());
if (unbufferedTransition.NextStateIsTerminal()) {
mTransition = unbufferedTransition;
return Some(mTransition.NextStateAsTerminal()); // Done!
return SetTransition(unbufferedTransition);
}
MOZ_ASSERT(mTransition.UnbufferedState() ==
unbufferedTransition.NextState());
aInput += toRead;
aLength -= toRead;
mToReadUnbuffered -= toRead;
mToReadUnbuffered -= aIterator.Length();
if (mToReadUnbuffered != 0) {
return Nothing(); // Need more input.
}
// We're done with the unbuffered read, so transition to the next state.
mTransition = aFunc(mTransition.NextState(), nullptr, 0);
if (mTransition.NextStateIsTerminal()) {
return Some(mTransition.NextStateAsTerminal()); // Done!
}
} else if (0 < mBuffer.length()) {
// We're continuing a buffered read.
MOZ_ASSERT(mToReadUnbuffered == 0,
"Shouldn't be continuing an unbuffered read and a buffered "
"read at the same time");
MOZ_ASSERT(mBuffer.length() < mTransition.Size(),
"Buffered more than we needed?");
size_t toRead = std::min(aLength, mTransition.Size() - mBuffer.length());
if (!mBuffer.append(aInput, toRead)) {
return Some(TerminalState::FAILURE);
}
aInput += toRead;
aLength -= toRead;
if (mBuffer.length() != mTransition.Size()) {
return Nothing(); // Need more input.
}
// We've buffered everything, so transition to the next state.
mTransition =
aFunc(mTransition.NextState(), mBuffer.begin(), mBuffer.length());
mBuffer.clear();
if (mTransition.NextStateIsTerminal()) {
return Some(mTransition.NextStateAsTerminal()); // Done!
return Nothing(); // Keep processing.
}
}
MOZ_ASSERT(mToReadUnbuffered == 0);
MOZ_ASSERT(mBuffer.empty());
// Process states as long as we continue to have enough input to do so.
while (mTransition.Size() <= aLength) {
size_t toRead = mTransition.Size();
if (mTransition.Buffering() == BufferingStrategy::BUFFERED) {
mTransition = aFunc(mTransition.NextState(), aInput, toRead);
} else {
MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::UNBUFFERED);
// Call aFunc with the unbuffered state to indicate that we're in the
// middle of an unbuffered read. We enforce that any state transition
// passed back to us is either a terminal state or takes us back to the
// unbuffered state.
LexerTransition<State> unbufferedTransition =
aFunc(mTransition.UnbufferedState(), aInput, toRead);
if (unbufferedTransition.NextStateIsTerminal()) {
mTransition = unbufferedTransition;
return Some(mTransition.NextStateAsTerminal()); // Done!
}
MOZ_ASSERT(mTransition.UnbufferedState() ==
unbufferedTransition.NextState());
// We're done with the unbuffered read, so transition to the next state.
mTransition = aFunc(mTransition.NextState(), nullptr, 0);
}
aInput += toRead;
aLength -= toRead;
if (mTransition.NextStateIsTerminal()) {
return Some(mTransition.NextStateAsTerminal()); // Done!
}
}
if (aLength == 0) {
// We finished right at a transition point. Just wait for more data.
return Nothing();
}
// If the next state is unbuffered, deliver what we can and then wait.
if (mTransition.Buffering() == BufferingStrategy::UNBUFFERED) {
LexerTransition<State> unbufferedTransition =
aFunc(mTransition.UnbufferedState(), aInput, aLength);
if (unbufferedTransition.NextStateIsTerminal()) {
mTransition = unbufferedTransition;
return Some(mTransition.NextStateAsTerminal()); // Done!
}
MOZ_ASSERT(mTransition.UnbufferedState() ==
unbufferedTransition.NextState());
mToReadUnbuffered = mTransition.Size() - aLength;
return Nothing(); // Need more input.
}
// If the next state is buffered, buffer what we can and then wait.
MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::BUFFERED);
if (!mBuffer.reserve(mTransition.Size())) {
return Some(TerminalState::FAILURE); // Done due to allocation failure.
}
if (!mBuffer.append(aInput, aLength)) {
return Some(TerminalState::FAILURE);
}
return Nothing(); // Need more input.
// We're done with the unbuffered read, so transition to the next state.
return SetTransition(aFunc(mTransition.NextState(), nullptr, 0));
}
template <typename Func>
Maybe<LexerResult> BufferedRead(SourceBufferIterator& aIterator, Func aFunc)
{
MOZ_ASSERT(mTransition.Buffering() == BufferingStrategy::BUFFERED);
MOZ_ASSERT(mToReadUnbuffered == 0,
"Buffered read at the same time as unbuffered read?");
MOZ_ASSERT(mBuffer.length() < mTransition.Size() ||
(mBuffer.length() == 0 && mTransition.Size() == 0),
"Buffered more than we needed?");
// If we have all the data, we don't actually need to buffer anything.
if (mBuffer.empty() && aIterator.Length() == mTransition.Size()) {
return SetTransition(aFunc(mTransition.NextState(),
aIterator.Data(),
aIterator.Length()));
}
// We do need to buffer, so make sure the buffer has enough capacity. We
// deliberately wait until we know for sure we need to buffer to call
// reserve() since it could require memory allocation.
if (!mBuffer.reserve(mTransition.Size())) {
return SetTransition(Transition::TerminateFailure());
}
// Append the new data we just got to the buffer.
if (!mBuffer.append(aIterator.Data(), aIterator.Length())) {
return SetTransition(Transition::TerminateFailure());
}
if (mBuffer.length() != mTransition.Size()) {
return Nothing(); // Keep processing.
}
// We've buffered everything, so transition to the next state.
return SetTransition(aFunc(mTransition.NextState(),
mBuffer.begin(),
mBuffer.length()));
}
Maybe<LexerResult> SetTransition(const LexerTransition<State>& aTransition)
{
mTransition = aTransition;
// Get rid of anything left over from the previous state.
mBuffer.clear();
mToReadUnbuffered = 0;
// If we reached a terminal state, let the caller know.
if (mTransition.NextStateIsTerminal()) {
return Some(LexerResult(mTransition.NextStateAsTerminal()));
}
// If we're entering an unbuffered state, record how long we'll stay in it.
if (mTransition.Buffering() == BufferingStrategy::UNBUFFERED) {
mToReadUnbuffered = mTransition.Size();
}
return Nothing(); // Keep processing.
}
private:
Vector<char, InlineBufferSize> mBuffer;
LexerTransition<State> mTransition;
size_t mToReadUnbuffered;

232
image/test/gtest/TestStreamingLexer.cpp
View File

@ -19,12 +19,13 @@ enum class TestState
};
void
CheckData(const char* aData, size_t aLength)
CheckLexedData(const char* aData, size_t aLength, size_t aExpectedLength)
{
EXPECT_TRUE(aLength == 3);
EXPECT_EQ(1, aData[0]);
EXPECT_EQ(2, aData[1]);
EXPECT_EQ(3, aData[2]);
EXPECT_TRUE(aLength == aExpectedLength);
for (size_t i = 0; i < aLength; ++i) {
EXPECT_EQ(aData[i], char((i % 3) + 1));
}
}
LexerTransition<TestState>
@ -32,16 +33,16 @@ DoLex(TestState aState, const char* aData, size_t aLength)
{
switch (aState) {
case TestState::ONE:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::To(TestState::TWO, 3);
case TestState::TWO:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::To(TestState::THREE, 3);
case TestState::THREE:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::TerminateSuccess();
default:
MOZ_CRASH("Unknown TestState");
MOZ_CRASH("Unexpected or unhandled TestState");
}
}
@ -51,20 +52,20 @@ DoLexWithUnbuffered(TestState aState, const char* aData, size_t aLength,
{
switch (aState) {
case TestState::ONE:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 3);
case TestState::UNBUFFERED:
EXPECT_TRUE(aLength <= 3);
EXPECT_TRUE(aUnbufferedVector.append(aData, aLength));
return Transition::ContinueUnbuffered(TestState::UNBUFFERED);
case TestState::TWO:
CheckData(aUnbufferedVector.begin(), aUnbufferedVector.length());
CheckLexedData(aUnbufferedVector.begin(), aUnbufferedVector.length(), 3);
return Transition::To(TestState::THREE, 3);
case TestState::THREE:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::TerminateSuccess();
default:
MOZ_CRASH("Unknown TestState");
MOZ_CRASH("Unexpected or unhandled TestState");
}
}
@ -73,12 +74,53 @@ DoLexWithUnbufferedTerminate(TestState aState, const char* aData, size_t aLength
{
switch (aState) {
case TestState::ONE:
CheckData(aData, aLength);
CheckLexedData(aData, aLength, 3);
return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 3);
case TestState::UNBUFFERED:
return Transition::TerminateSuccess();
default:
MOZ_CRASH("Unknown TestState");
MOZ_CRASH("Unexpected or unhandled TestState");
}
}
LexerTransition<TestState>
DoLexWithZeroLengthStates(TestState aState, const char* aData, size_t aLength)
{
switch (aState) {
case TestState::ONE:
EXPECT_TRUE(aLength == 0);
return Transition::To(TestState::TWO, 0);
case TestState::TWO:
EXPECT_TRUE(aLength == 0);
return Transition::To(TestState::THREE, 9);
case TestState::THREE:
CheckLexedData(aData, aLength, 9);
return Transition::TerminateSuccess();
default:
MOZ_CRASH("Unexpected or unhandled TestState");
}
}
LexerTransition<TestState>
DoLexWithZeroLengthStatesUnbuffered(TestState aState,
const char* aData,
size_t aLength)
{
switch (aState) {
case TestState::ONE:
EXPECT_TRUE(aLength == 0);
return Transition::ToUnbuffered(TestState::TWO, TestState::UNBUFFERED, 0);
case TestState::TWO:
EXPECT_TRUE(aLength == 0);
return Transition::To(TestState::THREE, 9);
case TestState::THREE:
CheckLexedData(aData, aLength, 9);
return Transition::TerminateSuccess();
case TestState::UNBUFFERED:
ADD_FAILURE() << "Should not enter zero-length unbuffered state";
return Transition::TerminateFailure();
default:
MOZ_CRASH("Unexpected or unhandled TestState");
}
}
@ -105,17 +147,13 @@ protected:
TEST_F(ImageStreamingLexer, ZeroLengthData)
{
// Test delivering a zero-length piece of data.
Maybe<TerminalState> result = mLexer.Lex(mData, 0, DoLex);
EXPECT_TRUE(result.isNothing());
}
// Test a zero-length input.
mSourceBuffer->Complete(NS_OK);
Maybe<TerminalState> result = mLexer.Lex(mIterator, mExpectNoResume, DoLex);
TEST_F(ImageStreamingLexer, SingleChunk)
{
// Test delivering all the data at once.
Maybe<TerminalState> result = mLexer.Lex(mData, sizeof(mData), DoLex);
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
EXPECT_EQ(Some(TerminalState::FAILURE), result);
}
TEST_F(ImageStreamingLexer, SingleChunkFromSourceBuffer)
@ -130,20 +168,6 @@ TEST_F(ImageStreamingLexer, SingleChunkFromSourceBuffer)
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
}
TEST_F(ImageStreamingLexer, SingleChunkWithUnbuffered)
{
Vector<char> unbufferedVector;
// Test delivering all the data at once.
Maybe<TerminalState> result =
mLexer.Lex(mData, sizeof(mData),
[&](TestState aState, const char* aData, size_t aLength) {
return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
});
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
}
TEST_F(ImageStreamingLexer, SingleChunkWithUnbufferedFromSourceBuffer)
{
Vector<char> unbufferedVector;
@ -162,21 +186,6 @@ TEST_F(ImageStreamingLexer, SingleChunkWithUnbufferedFromSourceBuffer)
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
}
TEST_F(ImageStreamingLexer, ChunkPerState)
{
// Test delivering in perfectly-sized chunks, one per state.
for (unsigned i = 0; i < 3; ++i) {
Maybe<TerminalState> result = mLexer.Lex(mData + 3 * i, 3, DoLex);
if (i == 2) {
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
} else {
EXPECT_TRUE(result.isNothing());
}
}
}
TEST_F(ImageStreamingLexer, ChunkPerStateFromSourceBuffer)
{
// Test delivering in perfectly-sized chunks, one per state.
@ -196,27 +205,6 @@ TEST_F(ImageStreamingLexer, ChunkPerStateFromSourceBuffer)
mSourceBuffer->Complete(NS_OK);
}
TEST_F(ImageStreamingLexer, ChunkPerStateWithUnbuffered)
{
Vector<char> unbufferedVector;
// Test delivering in perfectly-sized chunks, one per state.
for (unsigned i = 0; i < 3; ++i) {
Maybe<TerminalState> result =
mLexer.Lex(mData + 3 * i, 3,
[&](TestState aState, const char* aData, size_t aLength) {
return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
});
if (i == 2) {
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
} else {
EXPECT_TRUE(result.isNothing());
}
}
}
TEST_F(ImageStreamingLexer, ChunkPerStateWithUnbufferedFromSourceBuffer)
{
Vector<char> unbufferedVector;
@ -242,21 +230,6 @@ TEST_F(ImageStreamingLexer, ChunkPerStateWithUnbufferedFromSourceBuffer)
mSourceBuffer->Complete(NS_OK);
}
TEST_F(ImageStreamingLexer, OneByteChunks)
{
// Test delivering in one byte chunks.
for (unsigned i = 0; i < 9; ++i) {
Maybe<TerminalState> result = mLexer.Lex(mData + i, 1, DoLex);
if (i == 8) {
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
} else {
EXPECT_TRUE(result.isNothing());
}
}
}
TEST_F(ImageStreamingLexer, OneByteChunksFromSourceBuffer)
{
// Test delivering in one byte chunks.
@ -276,27 +249,6 @@ TEST_F(ImageStreamingLexer, OneByteChunksFromSourceBuffer)
mSourceBuffer->Complete(NS_OK);
}
TEST_F(ImageStreamingLexer, OneByteChunksWithUnbuffered)
{
Vector<char> unbufferedVector;
// Test delivering in one byte chunks.
for (unsigned i = 0; i < 9; ++i) {
Maybe<TerminalState> result =
mLexer.Lex(mData + i, 1,
[&](TestState aState, const char* aData, size_t aLength) {
return DoLexWithUnbuffered(aState, aData, aLength, unbufferedVector);
});
if (i == 8) {
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
} else {
EXPECT_TRUE(result.isNothing());
}
}
}
TEST_F(ImageStreamingLexer, OneByteChunksWithUnbufferedFromSourceBuffer)
{
Vector<char> unbufferedVector;
@ -322,28 +274,6 @@ TEST_F(ImageStreamingLexer, OneByteChunksWithUnbufferedFromSourceBuffer)
mSourceBuffer->Complete(NS_OK);
}
TEST_F(ImageStreamingLexer, TerminateSuccess)
{
// Test that Terminate is "sticky".
Maybe<TerminalState> result =
mLexer.Lex(mData, sizeof(mData),
[&](TestState aState, const char* aData, size_t aLength) {
EXPECT_TRUE(aState == TestState::ONE);
return Transition::TerminateSuccess();
});
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
result =
mLexer.Lex(mData, sizeof(mData),
[&](TestState aState, const char* aData, size_t aLength) {
EXPECT_TRUE(false); // Shouldn't get here.
return Transition::TerminateFailure();
});
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
}
TEST_F(ImageStreamingLexer, TerminateSuccessFromSourceBuffer)
{
mSourceBuffer->Append(mData, sizeof(mData));
@ -371,28 +301,6 @@ TEST_F(ImageStreamingLexer, TerminateSuccessFromSourceBuffer)
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
}
TEST_F(ImageStreamingLexer, TerminateFailure)
{
// Test that Terminate is "sticky".
Maybe<TerminalState> result =
mLexer.Lex(mData, sizeof(mData),
[&](TestState aState, const char* aData, size_t aLength) {
EXPECT_TRUE(aState == TestState::ONE);
return Transition::TerminateFailure();
});
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::FAILURE), result);
result =
mLexer.Lex(mData, sizeof(mData),
[&](TestState aState, const char* aData, size_t aLength) {
EXPECT_TRUE(false); // Shouldn't get here.
return Transition::TerminateFailure();
});
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::FAILURE), result);
}
TEST_F(ImageStreamingLexer, TerminateFailureFromSourceBuffer)
{
mSourceBuffer->Append(mData, sizeof(mData));
@ -420,22 +328,6 @@ TEST_F(ImageStreamingLexer, TerminateFailureFromSourceBuffer)
EXPECT_EQ(Some(TerminalState::FAILURE), result);
}
TEST_F(ImageStreamingLexer, TerminateUnbuffered)
{
// Test that Terminate works during an unbuffered read.
for (unsigned i = 0; i < 9; ++i) {
Maybe<TerminalState> result =
mLexer.Lex(mData + i, 1, DoLexWithUnbufferedTerminate);
if (i > 2) {
EXPECT_TRUE(result.isSome());
EXPECT_EQ(Some(TerminalState::SUCCESS), result);
} else {
EXPECT_TRUE(result.isNothing());
}
}
}
TEST_F(ImageStreamingLexer, TerminateUnbufferedFromSourceBuffer)
{
// Test that Terminate works during an unbuffered read.