[libFuzzer] add an experimental search heuristic flag -reduce_depth

llvm-svn: 319571

compiler-rt/lib/fuzzer/FuzzerDriver.cpp

@@ -566,6 +566,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.MaxTotalTimeSec = Flags.max_total_time;
   Options.DoCrossOver = Flags.cross_over;
   Options.MutateDepth = Flags.mutate_depth;
+  Options.ReduceDepth = Flags.reduce_depth;
   Options.UseCounters = Flags.use_counters;
   Options.UseIndirCalls = Flags.use_indir_calls;
   Options.UseMemmem = Flags.use_memmem;

compiler-rt/lib/fuzzer/FuzzerFlags.def

@@ -21,6 +21,8 @@ FUZZER_FLAG_INT(experimental_len_control, 0, "experimental flag")
 FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.")
 FUZZER_FLAG_INT(mutate_depth, 5,
             "Apply this number of consecutive mutations to each input.")
+FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. "
+                "Reduce depth if mutations lose unique features")
 FUZZER_FLAG_INT(shuffle, 1, "Shuffle inputs at startup")
 FUZZER_FLAG_INT(prefer_small, 1,
     "If 1, always prefer smaller inputs during the corpus shuffle.")
@@ -118,7 +120,7 @@ FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.")
 FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.")
 FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "if 2, close stderr; if 3, close both. "
-    "Be careful, this will also close e.g. asan's stderr/stdout.")
+    "Be careful, this will also close e.g. stderr of asan.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
 FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and "

compiler-rt/lib/fuzzer/FuzzerInternal.h

@@ -67,7 +67,7 @@ public:
 
   void ExecuteCallback(const uint8_t *Data, size_t Size);
   bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
-              InputInfo *II = nullptr);
+              InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
 
   // Merge Corpora[1:] into Corpora[0].
   void Merge(const Vector<std::string> &Corpora);

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

@@ -433,7 +433,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
 }
 
 bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
-                    InputInfo *II) {
+                    InputInfo *II, bool *FoundUniqFeatures) {
   if (!Size)
     return false;
 
@@ -451,6 +451,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
                              II->UniqFeatureSet.end(), Feature))
         FoundUniqFeaturesOfII++;
   });
+  if (FoundUniqFeatures)
+    *FoundUniqFeatures = FoundUniqFeaturesOfII;
   PrintPulseAndReportSlowInput(Data, Size);
   size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore;
   if (NumNewFeatures) {
@@ -642,11 +644,18 @@ void Fuzzer::MutateAndTestOne() {
     Size = NewSize;
     II.NumExecutedMutations++;
 
-    bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II);
+    bool FoundUniqFeatures = false;
+    bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II,
+                         &FoundUniqFeatures);
+    // Printf("FUF[%d] %d\n", i, FoundUniqFeatures);
     TryDetectingAMemoryLeak(CurrentUnitData, Size,
                             /*DuringInitialCorpusExecution*/ false);
-    if (NewCov)
+    if (NewCov) {
       ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size});
+      break;  // We will mutate this input more in the next rounds.
+    }
+    if (Options.ReduceDepth && !FoundUniqFeatures)
+        break;
   }
 }
 

compiler-rt/lib/fuzzer/FuzzerOptions.h

@@ -26,6 +26,7 @@ struct FuzzingOptions {
   int RssLimitMb = 0;
   bool DoCrossOver = true;
   int MutateDepth = 5;
+  bool ReduceDepth = false;
   bool UseCounters = false;
   bool UseIndirCalls = true;
   bool UseMemmem = true;