From 8848feb6a97c15a7a3fd02582ee8991429db787d Mon Sep 17 00:00:00 2001 From: Kostya Serebryany Date: Wed, 12 Jul 2017 22:20:04 +0000 Subject: [PATCH] [libFuzzer] refactoring in preparation for -reduce_inputs; NFC intended git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@307857 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Fuzzer/FuzzerDriver.cpp | 5 ++-- lib/Fuzzer/FuzzerFlags.def | 4 ++- lib/Fuzzer/FuzzerInternal.h | 6 ++-- lib/Fuzzer/FuzzerLoop.cpp | 56 +++++++++++++++++++------------------ lib/Fuzzer/FuzzerOptions.h | 1 + 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index 0453a7f443b..f88b5b7c2bb 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -265,7 +265,7 @@ int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { Unit U = FileToVector(InputFilePath); if (MaxLen && MaxLen < U.size()) U.resize(MaxLen); - F->RunOne(U.data(), U.size()); + F->ExecuteCallback(U.data(), U.size()); F->TryDetectingAMemoryLeak(U.data(), U.size(), true); return 0; } @@ -572,6 +572,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.UseCmp = Flags.use_cmp; Options.UseValueProfile = Flags.use_value_profile; Options.Shrink = Flags.shrink; + Options.ReduceInputs = Flags.reduce_inputs; Options.ShuffleAtStartUp = Flags.shuffle; Options.PreferSmall = Flags.prefer_small; Options.ReloadIntervalSec = Flags.reload; @@ -657,7 +658,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { size_t Size = SMR.ReadByteArraySize(); SMR.WriteByteArray(nullptr, 0); const Unit tmp(SMR.GetByteArray(), SMR.GetByteArray() + Size); - F->RunOne(tmp.data(), tmp.size()); + F->ExecuteCallback(tmp.data(), tmp.size()); SMR.PostServer(); } return 0; diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def index 7ff196c8fa9..5e70cbad3cf 100644 --- a/lib/Fuzzer/FuzzerFlags.def +++ b/lib/Fuzzer/FuzzerFlags.def @@ -65,7 +65,9 @@ FUZZER_FLAG_INT(use_memmem, 1, FUZZER_FLAG_INT(use_value_profile, 0, "Experimental. Use value profile to guide fuzzing.") FUZZER_FLAG_INT(use_cmp, 1, "Use CMP traces to guide mutations") -FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus elements.") +FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus inputs.") +FUZZER_FLAG_INT(reduce_inputs, 0, "Experimental. " + "Try to reduce the size of inputs wile preserving their full feature sets") FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" " this number of jobs in separate worker processes" " with stdout/stderr redirected to fuzz-JOB.log.") diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index 5f184c2316e..573923fddc8 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -65,7 +65,7 @@ public: static void StaticFileSizeExceedCallback(); void ExecuteCallback(const uint8_t *Data, size_t Size); - size_t RunOne(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false); // Merge Corpora[1:] into Corpora[0]. void Merge(const std::vector &Corpora); @@ -95,7 +95,7 @@ private: void InterruptCallback(); void MutateAndTestOne(); void ReportNewCoverage(InputInfo *II, const Unit &U); - size_t RunOne(const Unit &U) { return RunOne(U.data(), U.size()); } + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); @@ -142,6 +142,8 @@ private: size_t MaxInputLen = 0; size_t MaxMutationLen = 0; + std::vector FeatureSetTmp; + // Need to know our own thread. static thread_local bool IsMyThread; }; diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index fbf18357ede..88ec844ab44 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -348,11 +348,8 @@ void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (U.size() > MaxSize) U.resize(MaxSize); if (!Corpus.HasUnit(U)) { - if (size_t NumFeatures = RunOne(U)) { - CheckExitOnSrcPosOrItem(); - Corpus.AddToCorpus(U, NumFeatures); + if (RunOne(U.data(), U.size())) Reloaded = true; - } } } if (Reloaded) @@ -377,10 +374,7 @@ void Fuzzer::ShuffleAndMinimize(UnitVector *InitialCorpus) { ExecuteCallback(&dummy, 0); for (const auto &U : *InitialCorpus) { - if (size_t NumFeatures = RunOne(U)) { - CheckExitOnSrcPosOrItem(); - Corpus.AddToCorpus(U, NumFeatures); - } + RunOne(U.data(), U.size()); TryDetectingAMemoryLeak(U.data(), U.size(), /*DuringInitialCorpusExecution*/ true); } @@ -392,18 +386,7 @@ void Fuzzer::ShuffleAndMinimize(UnitVector *InitialCorpus) { } } -size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) { - if (!Size) return 0; - TotalNumberOfRuns++; - - ExecuteCallback(Data, Size); - - size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - Corpus.AddFeature(Feature, Size, Options.Shrink); - }); - size_t NumUpdatesAfter = Corpus.NumFeatureUpdates(); - +void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { auto TimeOfUnit = duration_cast(UnitStopTime - UnitStartTime).count(); if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && @@ -415,7 +398,27 @@ size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) { Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); } - return NumUpdatesAfter - NumUpdatesBefore; +} + +bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile) { + if (!Size) return false; + + ExecuteCallback(Data, Size); + + FeatureSetTmp.clear(); + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + Corpus.AddFeature(Feature, Size, Options.Shrink); + if (Options.ReduceInputs) + FeatureSetTmp.push_back(Feature); + }); + PrintPulseAndReportSlowInput(Data, Size); + size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; + if (NumNewFeatures) { + CheckExitOnSrcPosOrItem(); + Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile); + } + return NumNewFeatures > 0; } size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const { @@ -443,6 +446,7 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { } void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TotalNumberOfRuns++; assert(InFuzzingThread()); if (SMR.IsClient()) SMR.WriteByteArray(Data, Size); @@ -595,12 +599,9 @@ void Fuzzer::MutateAndTestOne() { if (i == 0) StartTraceRecording(); II.NumExecutedMutations++; - if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) { - Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures, - /*MayDeleteFile=*/true); + if (RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true)) ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); - CheckExitOnSrcPosOrItem(); - } + StopTraceRecording(); TryDetectingAMemoryLeak(CurrentUnitData, Size, /*DuringInitialCorpusExecution*/ false); @@ -638,7 +639,8 @@ void Fuzzer::MinimizeCrashLoop(const Unit &U) { for (int i = 0; i < Options.MutateDepth; i++) { size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); assert(NewSize > 0 && NewSize <= MaxMutationLen); - RunOne(CurrentUnitData, NewSize); + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); TryDetectingAMemoryLeak(CurrentUnitData, NewSize, /*DuringInitialCorpusExecution*/ false); } diff --git a/lib/Fuzzer/FuzzerOptions.h b/lib/Fuzzer/FuzzerOptions.h index b1366789be0..9500235e2b1 100644 --- a/lib/Fuzzer/FuzzerOptions.h +++ b/lib/Fuzzer/FuzzerOptions.h @@ -32,6 +32,7 @@ struct FuzzingOptions { bool UseCmp = false; bool UseValueProfile = false; bool Shrink = false; + bool ReduceInputs = false; int ReloadIntervalSec = 1; bool ShuffleAtStartUp = true; bool PreferSmall = true;