[libFuzzer] add two experimental flags to make corpus merging more scalable: -save_coverage_summary/-load_coverage_summary. This is still WIP, the documentation will come later if these flags survive

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298548 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Fuzzer/FuzzerDriver.cpp

@@ -601,7 +601,9 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
     if (Flags.merge_control_file)
       F->CrashResistantMergeInternalStep(Flags.merge_control_file);
     else
-      F->CrashResistantMerge(Args, *Inputs);
+      F->CrashResistantMerge(Args, *Inputs,
+                             Flags.load_coverage_summary,
+                             Flags.save_coverage_summary);
     exit(0);
   }
 

lib/Fuzzer/FuzzerFlags.def

@@ -39,6 +39,13 @@ FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
   "merged into the 1-st corpus. Only interesting units will be taken. "
   "This flag can be used to minimize a corpus.")
 FUZZER_FLAG_STRING(merge_control_file, "internal flag")
+FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:"
+                   " save coverage summary to a given file."
+                   " Used with -merge=1")
+FUZZER_FLAG_STRING(load_coverage_summary, "Experimental:"
+                   " load coverage summary from a given file."
+                   " Treat this coverage as belonging to the first corpus. "
+                   " Used with -merge=1")
 FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided"
   " crash input. Use with -runs=N or -max_total_time=N to limit "
   "the number attempts")

lib/Fuzzer/FuzzerInternal.h

@@ -70,7 +70,9 @@ public:
   // Merge Corpora[1:] into Corpora[0].
   void Merge(const std::vector<std::string> &Corpora);
   void CrashResistantMerge(const std::vector<std::string> &Args,
-                           const std::vector<std::string> &Corpora);
+                           const std::vector<std::string> &Corpora,
+                           const char *CoverageSummaryInputPathOrNull,
+                           const char *CoverageSummaryOutputPathOrNull);
   void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();

lib/Fuzzer/FuzzerMerge.cpp

@@ -122,10 +122,11 @@ size_t Merger::ApproximateMemoryConsumption() const  {
 
 // Decides which files need to be merged (add thost to NewFiles).
 // Returns the number of new features added.
-size_t Merger::Merge(std::vector<std::string> *NewFiles) {
+size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
+                     std::vector<std::string> *NewFiles) {
   NewFiles->clear();
   assert(NumFilesInFirstCorpus <= Files.size());
-  std::set<uint32_t> AllFeatures;
+  std::set<uint32_t> AllFeatures(InitialFeatures);
 
   // What features are in the initial corpus?
   for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
@@ -167,6 +168,42 @@ size_t Merger::Merge(std::vector<std::string> *NewFiles) {
   return AllFeatures.size() - InitialNumFeatures;
 }
 
+void Merger::PrintSummary(std::ostream &OS) {
+  for (auto &File : Files) {
+    OS << std::hex;
+    OS << File.Name << " size: " << File.Size << " features: ";
+    for (auto Feature : File.Features)
+      OS << " " << Feature;
+    OS << "\n";
+  }
+}
+
+std::set<uint32_t> Merger::AllFeatures() const {
+  std::set<uint32_t> S;
+  for (auto &File : Files)
+    S.insert(File.Features.begin(), File.Features.end());
+  return S;
+}
+
+std::set<uint32_t> Merger::ParseSummary(std::istream &IS) {
+  std::string Line, Tmp;
+  std::set<uint32_t> Res;
+  while (std::getline(IS, Line, '\n')) {
+    size_t N;
+    std::istringstream ISS1(Line);
+    ISS1 >> Tmp;  // Name
+    ISS1 >> Tmp;  // size:
+    assert(Tmp == "size:" && "Corrupt summary file");
+    ISS1 >> std::hex;
+    ISS1 >> N;    // File Size
+    ISS1 >> Tmp;  // features:
+    assert(Tmp == "features:" && "Corrupt summary file");
+    while (ISS1 >> std::hex >> N)
+      Res.insert(N);
+  }
+  return Res;
+}
+
 // Inner process. May crash if the target crashes.
 void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
   Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str());
@@ -217,7 +254,9 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
 
 // Outer process. Does not call the target code and thus sohuld not fail.
 void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args,
-                                 const std::vector<std::string> &Corpora) {
+                                 const std::vector<std::string> &Corpora,
+                                 const char *CoverageSummaryInputPathOrNull,
+                                 const char *CoverageSummaryOutputPathOrNull) {
   if (Corpora.size() <= 1) {
     Printf("Merge requires two or more corpus dirs\n");
     return;
@@ -273,8 +312,21 @@ void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args,
   IF.close();
   Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n",
          M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
+  if (CoverageSummaryOutputPathOrNull) {
+    Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n",
+           M.Files.size(), CoverageSummaryOutputPathOrNull);
+    std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull);
+    M.PrintSummary(SummaryOut);
+  }
   std::vector<std::string> NewFiles;
-  size_t NumNewFeatures = M.Merge(&NewFiles);
+  std::set<uint32_t> InitialFeatures;
+  if (CoverageSummaryInputPathOrNull) {
+    std::ifstream SummaryIn(CoverageSummaryInputPathOrNull);
+    InitialFeatures = M.ParseSummary(SummaryIn);
+    Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n",
+           CoverageSummaryInputPathOrNull, InitialFeatures.size());
+  }
+  size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles);
   Printf("MERGE-OUTER: %zd new files with %zd new features added\n",
          NewFiles.size(), NumNewFeatures);
   for (auto &F: NewFiles)

lib/Fuzzer/FuzzerMerge.h

@@ -43,6 +43,9 @@
 #include "FuzzerDefs.h"
 
 #include <istream>
+#include <ostream>
+#include <set>
+#include <vector>
 
 namespace fuzzer {
 
@@ -61,8 +64,15 @@ struct Merger {
   bool Parse(std::istream &IS, bool ParseCoverage);
   bool Parse(const std::string &Str, bool ParseCoverage);
   void ParseOrExit(std::istream &IS, bool ParseCoverage);
-  size_t Merge(std::vector<std::string> *NewFiles);
+  void PrintSummary(std::ostream &OS);
+  std::set<uint32_t> ParseSummary(std::istream &IS);
+  size_t Merge(const std::set<uint32_t> &InitialFeatures,
+               std::vector<std::string> *NewFiles);
+  size_t Merge(std::vector<std::string> *NewFiles) {
+    return Merge({}, NewFiles);
+  }
   size_t ApproximateMemoryConsumption() const;
+  std::set<uint32_t> AllFeatures() const;
 };
 
 }  // namespace fuzzer

lib/Fuzzer/test/FuzzerUnittest.cpp

@@ -14,6 +14,7 @@
 #include "gtest/gtest.h"
 #include <memory>
 #include <set>
+#include <sstream>
 
 using namespace fuzzer;
 
@@ -636,7 +637,10 @@ static void Merge(const std::string &Input,
   Merger M;
   std::vector<std::string> NewFiles;
   EXPECT_TRUE(M.Parse(Input, true));
+  std::stringstream SS;
+  M.PrintSummary(SS);
   EXPECT_EQ(NumNewFeatures, M.Merge(&NewFiles));
+  EXPECT_EQ(M.AllFeatures(), M.ParseSummary(SS));
   EQ(NewFiles, Result);
 }
 
@@ -706,6 +710,16 @@ TEST(Merge, Good) {
   EQ(M.Files[2].Features, {1, 3, 6});
   EXPECT_EQ(3U, M.Merge(&NewFiles));
   EQ(NewFiles, {"B"});
+
+  // Same as the above, but with InitialFeatures.
+  EXPECT_TRUE(M.Parse("2\n0\nB\nC\n"
+                        "STARTED 0 1001\nDONE 0 4 5 6 \n"
+                        "STARTED 1 1002\nDONE 1 6 1 3\n"
+                        "", true));
+  EQ(M.Files[0].Features, {4, 5, 6});
+  EQ(M.Files[1].Features, {1, 3, 6});
+  EXPECT_EQ(3U, M.Merge({1, 2, 3}, &NewFiles));
+  EQ(NewFiles, {"B"});
 }
 
 TEST(Merge, Merge) {

lib/Fuzzer/test/merge-summary.test

@@ -0,0 +1,15 @@
+RUN: rm -rf %t/T1 %t/T2
+RUN: mkdir -p %t/T0 %t/T1 %t/T2
+RUN: echo ...Z.. > %t/T2/1
+RUN: echo ....E. > %t/T2/2
+RUN: echo .....R > %t/T2/3
+RUN: echo F..... > %t/T2/a
+RUN: echo .U.... > %t/T2/b
+RUN: echo ..Z... > %t/T2/c
+
+RUN: LLVMFuzzer-FullCoverageSetTest -merge=1 %t/T1 %t/T2 -save_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=SAVE_SUMMARY
+SAVE_SUMMARY: MERGE-OUTER: writing coverage summary for 6 files to {{.*}}SUMMARY
+RUN: rm %t/T1/*
+RUN: LLVMFuzzer-FullCoverageSetTest -merge=1 %t/T1 %t/T2 -load_coverage_summary=%t/SUMMARY 2>&1 | FileCheck %s --check-prefix=LOAD_SUMMARY
+LOAD_SUMMARY: MERGE-OUTER: coverage summary loaded from {{.*}}SUMMAR
+LOAD_SUMMARY: MERGE-OUTER: 0 new files with 0 new features added