RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-02-27 15:57:30 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[libFuzzer] replace vector with a simpler data structure in the Dictionaries to avoid memory allocations on hot path

Browse Source 
llvm-svn: 257985
This commit is contained in:
Kostya Serebryany 2016-01-16 03:53:32 +00:00
parent 3f93ca1561
commit 0ae292d42e
6 changed files with 84 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/Fuzzer/FuzzerDriver.cpp
View File

@ -301,7 +301,8 @@ int FuzzerDriver(const std::vector<std::string> &Args,
Fuzzer F(USF, Options);
for (auto &U: Dictionary)
USF.GetMD().AddWordToManualDictionary(U);
if (U.size() <= Word::GetMaxSize())
USF.GetMD().AddWordToManualDictionary(Word(U.data(), U.size()));
// Timer
if (Flags.timeout > 0)

40
lib/Fuzzer/FuzzerInternal.h
View File

@ -18,6 +18,7 @@
#include <cstddef>
#include <cstdlib>
#include <string>
#include <string.h>
#include <vector>
#include <unordered_set>
@ -27,6 +28,40 @@ namespace fuzzer {
using namespace std::chrono;
typedef std::vector<uint8_t> Unit;
// A simple POD sized array of bytes.
template<size_t kMaxSize>
class FixedWord {
public:
FixedWord() : Size(0) {}
FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); }
void Set(const uint8_t *B, uint8_t S) {
assert(S <= kMaxSize);
memcpy(Data, B, S);
Size = S;
}
bool operator == (const FixedWord<kMaxSize> &w) const {
return Size == w.Size && 0 == memcmp(Data, w.Data, Size);
}
bool operator < (const FixedWord<kMaxSize> &w) const {
if (Size != w.Size) return Size < w.Size;
return memcmp(Data, w.Data, Size) < 0;
}
static size_t GetMaxSize() { return kMaxSize; }
const uint8_t *data() const { return Data; }
uint8_t size() const { return Size; }
private:
uint8_t Size;
uint8_t Data[kMaxSize];
};
typedef FixedWord<27> Word; // 28 bytes.
std::string FileToString(const std::string &Path);
Unit FileToVector(const std::string &Path);
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
@ -43,6 +78,7 @@ void PrintHexArray(const uint8_t *Data, size_t Size,
const char *PrintAfter = "");
void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter = "");
void PrintASCII(const Unit &U, const char *PrintAfter = "");
void PrintASCII(const Word &W, const char *PrintAfter = "");
std::string Hash(const Unit &U);
void SetTimer(int Seconds);
std::string Base64(const Unit &U);
@ -118,9 +154,9 @@ class MutationDispatcher {
size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2,
size_t Size2, uint8_t *Out, size_t MaxOutSize);
void AddWordToManualDictionary(const Unit &Word);
void AddWordToManualDictionary(const Word &W);
void AddWordToAutoDictionary(const Unit &Word, size_t PositionHint);
void AddWordToAutoDictionary(const Word &W, size_t PositionHint);
void ClearAutoDictionary();
void PrintRecommendedDictionary();

51
lib/Fuzzer/FuzzerMutate.cpp
View File

@ -23,16 +23,15 @@ struct Mutator {
};
struct DictionaryEntry {
Unit Word;
Word W;
size_t PositionHint;
};
struct Dictionary : public std::vector<DictionaryEntry>{
bool ContainsWord(const Unit &W) const {
bool ContainsWord(const Word &W) const {
return end() !=
std::find_if(begin(), end(), [&](const DictionaryEntry &DE) {
return DE.Word == W;
});
std::find_if(begin(), end(),
[&](const DictionaryEntry &DE) { return DE.W == W; });
}
};
@ -161,20 +160,20 @@ size_t MutationDispatcher::Impl::AddWordFromDictionary(
size_t MaxSize) {
if (D.empty()) return 0;
const DictionaryEntry &DE = D[Rand(D.size())];
const Unit &Word = DE.Word;
const Word &W = DE.W;
size_t PositionHint = DE.PositionHint;
bool UsePositionHint = PositionHint != std::numeric_limits<size_t>::max() &&
PositionHint + Word.size() < Size && Rand.RandBool();
if (Rand.RandBool()) { // Insert Word.
if (Size + Word.size() > MaxSize) return 0;
PositionHint + W.size() < Size && Rand.RandBool();
if (Rand.RandBool()) { // Insert W.
if (Size + W.size() > MaxSize) return 0;
size_t Idx = UsePositionHint ? PositionHint : Rand(Size + 1);
memmove(Data + Idx + Word.size(), Data + Idx, Size - Idx);
memcpy(Data + Idx, Word.data(), Word.size());
Size += Word.size();
} else { // Overwrite some bytes with Word.
if (Word.size() > Size) return 0;
size_t Idx = UsePositionHint ? PositionHint : Rand(Size - Word.size());
memcpy(Data + Idx, Word.data(), Word.size());
memmove(Data + Idx + W.size(), Data + Idx, Size - Idx);
memcpy(Data + Idx, W.data(), W.size());
Size += W.size();
} else { // Overwrite some bytes with W.
if (W.size() > Size) return 0;
size_t Idx = UsePositionHint ? PositionHint : Rand(Size - W.size());
memcpy(Data + Idx, W.data(), W.size());
}
CurrentDictionaryEntrySequence.push_back(DE);
return Size;
@ -238,16 +237,16 @@ void MutationDispatcher::StartMutationSequence() {
void MutationDispatcher::RecordSuccessfulMutationSequence() {
for (auto &DE : MDImpl->CurrentDictionaryEntrySequence)
// Linear search is fine here as this happens seldom.
if (!MDImpl->PersistentAutoDictionary.ContainsWord(DE.Word))
if (!MDImpl->PersistentAutoDictionary.ContainsWord(DE.W))
MDImpl->PersistentAutoDictionary.push_back(
{DE.Word, std::numeric_limits<size_t>::max()});
{DE.W, std::numeric_limits<size_t>::max()});
}
void MutationDispatcher::PrintRecommendedDictionary() {
std::vector<Unit> V;
std::vector<Word> V;
for (auto &DE : MDImpl->PersistentAutoDictionary)
if (!MDImpl->ManualDictionary.ContainsWord(DE.Word))
V.push_back(DE.Word);
if (!MDImpl->ManualDictionary.ContainsWord(DE.W))
V.push_back(DE.W);
if (V.empty()) return;
Printf("###### Recommended dictionary. ######\n");
for (auto &U: V) {
@ -265,7 +264,7 @@ void MutationDispatcher::PrintMutationSequence() {
Printf(" DE: ");
for (auto &DE : MDImpl->CurrentDictionaryEntrySequence) {
Printf("\"");
PrintASCII(DE.Word, "\"-");
PrintASCII(DE.W, "\"-");
}
}
}
@ -299,16 +298,16 @@ void MutationDispatcher::SetCorpus(const std::vector<Unit> *Corpus) {
MDImpl->SetCorpus(Corpus);
}
void MutationDispatcher::AddWordToManualDictionary(const Unit &Word) {
void MutationDispatcher::AddWordToManualDictionary(const Word &W) {
MDImpl->ManualDictionary.push_back(
{Word, std::numeric_limits<size_t>::max()});
{W, std::numeric_limits<size_t>::max()});
}
void MutationDispatcher::AddWordToAutoDictionary(const Unit &Word,
void MutationDispatcher::AddWordToAutoDictionary(const Word &W,
size_t PositionHint) {
static const size_t kMaxAutoDictSize = 1 << 14;
if (MDImpl->TempAutoDictionary.size() >= kMaxAutoDictSize) return;
MDImpl->TempAutoDictionary.push_back({Word, PositionHint});
MDImpl->TempAutoDictionary.push_back({W, PositionHint});
}
void MutationDispatcher::ClearAutoDictionary() {

23
lib/Fuzzer/FuzzerTraceState.cpp
View File

@ -164,14 +164,10 @@ struct LabelRange {
// For now, very simple: put Size bytes of Data at position Pos.
struct TraceBasedMutation {
static const size_t kMaxSize = 28;
uint32_t Pos : 24;
uint32_t Size : 8;
uint8_t Data[kMaxSize];
uint32_t Pos;
Word W;
};
const size_t TraceBasedMutation::kMaxSize;
// Declared as static globals for faster checks inside the hooks.
static bool RecordingTraces = false;
static bool RecordingMemcmp = false;
@ -223,12 +219,11 @@ class TraceState {
RecordingMemcmp = false;
for (size_t i = 0; i < NumMutations; i++) {
auto &M = Mutations[i];
Unit U(M.Data, M.Data + M.Size);
if (Options.Verbosity >= 2) {
AutoDictUnitCounts[U]++;
AutoDictUnitCounts[M.W]++;
AutoDictAdds++;
if ((AutoDictAdds & (AutoDictAdds - 1)) == 0) {
typedef std::pair<size_t, Unit> CU;
typedef std::pair<size_t, Word> CU;
std::vector<CU> CountedUnits;
for (auto &I : AutoDictUnitCounts)
CountedUnits.push_back(std::make_pair(I.second, I.first));
@ -242,17 +237,15 @@ class TraceState {
}
}
}
USF.GetMD().AddWordToAutoDictionary(U, M.Pos);
USF.GetMD().AddWordToAutoDictionary(M.W, M.Pos);
}
}
void AddMutation(uint32_t Pos, uint32_t Size, const uint8_t *Data) {
if (NumMutations >= kMaxMutations) return;
assert(Size <= TraceBasedMutation::kMaxSize);
auto &M = Mutations[NumMutations++];
M.Pos = Pos;
M.Size = Size;
memcpy(M.Data, Data, Size);
M.W.Set(Data, Size);
}
void AddMutation(uint32_t Pos, uint32_t Size, uint64_t Data) {
@ -274,7 +267,7 @@ class TraceState {
const Fuzzer::FuzzingOptions &Options;
uint8_t **CurrentUnitData;
size_t *CurrentUnitSize;
std::map<Unit, size_t> AutoDictUnitCounts;
std::map<Word, size_t> AutoDictUnitCounts;
size_t AutoDictAdds = 0;
static thread_local bool IsMyThread;
};
@ -423,7 +416,7 @@ void TraceState::TraceCmpCallback(uintptr_t PC, size_t CmpSize, size_t CmpType,
void TraceState::TraceMemcmpCallback(size_t CmpSize, const uint8_t *Data1,
const uint8_t *Data2) {
if (!RecordingMemcmp || !IsMyThread) return;
CmpSize = std::min(CmpSize, TraceBasedMutation::kMaxSize);
CmpSize = std::min(CmpSize, Word::GetMaxSize());
int Added2 = TryToAddDesiredData(Data1, Data2, CmpSize);
int Added1 = TryToAddDesiredData(Data2, Data1, CmpSize);
if ((Added1 || Added2) && Options.Verbosity >= 3) {

8
lib/Fuzzer/FuzzerUtil.cpp
View File

@ -49,10 +49,12 @@ void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) {
Printf("%s", PrintAfter);
}
void PrintASCII(const Word &W, const char *PrintAfter) {
PrintASCII(W.data(), W.size(), PrintAfter);
}
void PrintASCII(const Unit &U, const char *PrintAfter) {
for (auto X : U)
PrintASCIIByte(X);
Printf("%s", PrintAfter);
PrintASCII(U.data(), U.size(), PrintAfter);
}
std::string Hash(const Unit &U) {

12
lib/Fuzzer/test/FuzzerUnittest.cpp
View File

@ -247,8 +247,8 @@ void TestAddWordFromDictionary(Mutator M, int NumIter) {
MutationDispatcher MD(Rand);
uint8_t Word1[4] = {0xAA, 0xBB, 0xCC, 0xDD};
uint8_t Word2[3] = {0xFF, 0xEE, 0xEF};
MD.AddWordToManualDictionary(Unit(Word1, Word1 + sizeof(Word1)));
MD.AddWordToManualDictionary(Unit(Word2, Word2 + sizeof(Word2)));
MD.AddWordToManualDictionary(Word(Word1, sizeof(Word1)));
MD.AddWordToManualDictionary(Word(Word2, sizeof(Word2)));
int FoundMask = 0;
uint8_t CH0[7] = {0x00, 0x11, 0x22, 0xAA, 0xBB, 0xCC, 0xDD};
uint8_t CH1[7] = {0x00, 0x11, 0xAA, 0xBB, 0xCC, 0xDD, 0x22};
@ -285,16 +285,16 @@ TEST(FuzzerMutate, AddWordFromDictionary2) {
void TestAddWordFromDictionaryWithHint(Mutator M, int NumIter) {
FuzzerRandomLibc Rand(0);
MutationDispatcher MD(Rand);
uint8_t Word[] = {0xAA, 0xBB, 0xCC, 0xDD, 0xFF, 0xEE, 0xEF};
uint8_t W[] = {0xAA, 0xBB, 0xCC, 0xDD, 0xFF, 0xEE, 0xEF};
size_t PosHint = 7777;
MD.AddWordToAutoDictionary(Unit(Word, Word + sizeof(Word)), PosHint);
MD.AddWordToAutoDictionary(Word(W, sizeof(W)), PosHint);
int FoundMask = 0;
for (int i = 0; i < NumIter; i++) {
uint8_t T[10000];
memset(T, 0, sizeof(T));
size_t NewSize = (MD.*M)(T, 9000, 10000);
if (NewSize >= PosHint + sizeof(Word) &&
!memcmp(Word, T + PosHint, sizeof(Word)))
if (NewSize >= PosHint + sizeof(W) &&
!memcmp(W, T + PosHint, sizeof(W)))
FoundMask = 1;
}
EXPECT_EQ(FoundMask, 1);