Add profile summary support for sample profile.

Differential Revision: http://reviews.llvm.org/D17178



git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@261304 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/ProfileData/InstrProf.h

@@ -681,14 +681,6 @@ struct Header {
   uint64_t HashOffset;
 };
 
-static const uint32_t SummaryCutoffs[] = {
-    10000,  /*  1% */
-    100000, /* 10% */
-    200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000,
-    800000, 900000, 950000, 990000, 999000, 999900, 999990, 999999};
-static const uint32_t NumSummaryCutoffs =
-    sizeof(SummaryCutoffs) / sizeof(*SummaryCutoffs);
-
 // Profile summary data recorded in the profile data file in indexed
 // format. It is introduced in version 4. The summary data follows
 // right after the profile file header.

include/llvm/ProfileData/ProfileCommon.h

@@ -24,6 +24,9 @@ namespace llvm {
 namespace IndexedInstrProf {
 struct Summary;
 }
+namespace sampleprof {
+class FunctionSamples;
+}
 struct InstrProfRecord;
 // The profile summary is one or more (Cutoff, MinCount, NumCounts) triplets.
 // The semantics of counts depend on the type of profile. For instrumentation
@@ -55,12 +58,18 @@ protected:
       : DetailedSummaryCutoffs(Cutoffs), TotalCount(0), MaxCount(0),
         NumCounts(0) {}
   ProfileSummary() : TotalCount(0), MaxCount(0), NumCounts(0) {}
+  ProfileSummary(std::vector<ProfileSummaryEntry> DetailedSummary,
+                 uint64_t TotalCount, uint64_t MaxCount, uint32_t NumCounts)
+      : DetailedSummary(DetailedSummary), TotalCount(TotalCount),
+        MaxCount(MaxCount), NumCounts(NumCounts) {}
   inline void addCount(uint64_t Count);
 
 public:
   static const int Scale = 1000000;
   inline std::vector<ProfileSummaryEntry> &getDetailedSummary();
   void computeDetailedSummary();
+  /// \brief A vector of useful cutoff values for detailed summary.
+  static const std::vector<uint32_t> DefaultCutoffs;
 };
 
 class InstrProfSummary : public ProfileSummary {
@@ -83,6 +92,28 @@ public:
   uint64_t getMaxInternalBlockCount() { return MaxInternalBlockCount; }
 };
 
+class SampleProfileSummary : public ProfileSummary {
+  uint64_t MaxHeadSamples;
+  uint32_t NumFunctions;
+
+public:
+  uint32_t getNumLinesWithSamples() { return NumCounts; }
+  uint64_t getTotalSamples() { return TotalCount; }
+  uint32_t getNumFunctions() { return NumFunctions; }
+  uint64_t getMaxHeadSamples() { return MaxHeadSamples; }
+  uint64_t getMaxSamplesPerLine() { return MaxCount; }
+  void addRecord(const sampleprof::FunctionSamples &FS);
+  SampleProfileSummary(std::vector<uint32_t> Cutoffs)
+      : ProfileSummary(Cutoffs), MaxHeadSamples(0), NumFunctions(0) {}
+  SampleProfileSummary(uint64_t TotalSamples, uint64_t MaxSamplesPerLine,
+                       uint64_t MaxHeadSamples, int32_t NumLinesWithSamples,
+                       uint32_t NumFunctions,
+                       std::vector<ProfileSummaryEntry> DetailedSummary)
+      : ProfileSummary(DetailedSummary, TotalSamples, MaxSamplesPerLine,
+                       NumLinesWithSamples),
+        MaxHeadSamples(MaxHeadSamples), NumFunctions(NumFunctions) {}
+};
+
 // This is called when a count is seen in the profile.
 void ProfileSummary::addCount(uint64_t Count) {
   TotalCount += Count;

include/llvm/ProfileData/SampleProf.h

@@ -74,7 +74,7 @@ static inline uint64_t SPMagic() {
          uint64_t('2') << (64 - 56) | uint64_t(0xff);
 }
 
-static inline uint64_t SPVersion() { return 102; }
+static inline uint64_t SPVersion() { return 103; }
 
 /// Represents the relative location of an instruction.
 ///

include/llvm/ProfileData/SampleProfReader.h

@@ -129,6 +129,30 @@
 // VERSION (uint32_t)
 //    File format version number computed by SPVersion()
 //
+// SUMMARY
+//    TOTAL_COUNT (uint64_t)
+//        Total number of samples in the profile.
+//    MAX_COUNT (uint64_t)
+//        Maximum value of samples on a line.
+//    MAX_HEAD_SAMPLES (uint64_t)
+//        Maximum number of head samples.
+//    NUM_COUNTS (uint64_t)
+//        Number of lines with samples.
+//    NUM_FUNCTIONS (uint64_t)
+//        Number of functions with samples.
+//    NUM_DETAILED_SUMMARY_ENTRIES (size_t)
+//        Number of entries in detailed summary
+//    DETAILED_SUMMARY
+//        A list of detailed summary entry. Each entry consists of
+//        CUTOFF (uint32_t)
+//            Required percentile of total sample count expressed as a fraction
+//            multiplied by 1000000.
+//        MIN_COUNT (uint64_t)
+//            The minimum number of samples required to reach the target
+//            CUTOFF.
+//        NUM_COUNTS (uint64_t)
+//            Number of samples to get to the desrired percentile.
+//
 // NAME TABLE
 //    SIZE (uint32_t)
 //        Number of entries in the name table.
@@ -190,6 +214,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/ProfileData/ProfileCommon.h"
 #include "llvm/ProfileData/SampleProf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -270,6 +295,9 @@ public:
   static ErrorOr<std::unique_ptr<SampleProfileReader>>
   create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C);
 
+  /// \brief Return the profile summary.
+  SampleProfileSummary &getSummary() { return *(Summary.get()); }
+
 protected:
   /// \brief Map every function to its associated profile.
   ///
@@ -283,6 +311,12 @@ protected:
 
   /// \brief Memory buffer holding the profile file.
   std::unique_ptr<MemoryBuffer> Buffer;
+
+  /// \brief Profile summary information.
+  std::unique_ptr<SampleProfileSummary> Summary;
+
+  /// \brief Compute summary for this profile.
+  void computeSummary();
 };
 
 class SampleProfileReaderText : public SampleProfileReader {
@@ -348,6 +382,12 @@ protected:
 
   /// Function name table.
   std::vector<StringRef> NameTable;
+
+private:
+  std::error_code readSummaryEntry(std::vector<ProfileSummaryEntry> &Entries);
+
+  /// \brief Read profile summary.
+  std::error_code readSummary();
 };
 
 typedef SmallVector<FunctionSamples *, 10> InlineCallStack;

include/llvm/ProfileData/SampleProfWriter.h

@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ProfileData/ProfileCommon.h"
 #include "llvm/ProfileData/SampleProf.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
@@ -42,7 +43,6 @@ public:
   std::error_code write(const StringMap<FunctionSamples> &ProfileMap) {
     if (std::error_code EC = writeHeader(ProfileMap))
       return EC;
-
     for (const auto &I : ProfileMap) {
       StringRef FName = I.first();
       const FunctionSamples &Profile = I.second;
@@ -75,6 +75,12 @@ protected:
 
   /// \brief Output stream where to emit the profile to.
   std::unique_ptr<raw_ostream> OutputStream;
+
+  /// \brief Profile summary.
+  std::unique_ptr<SampleProfileSummary> Summary;
+
+  /// \brief Compute summary for this profile.
+  void computeSummary(const StringMap<FunctionSamples> &ProfileMap);
 };
 
 /// \brief Sample-based profile writer (text format).
@@ -113,6 +119,7 @@ protected:
 
   std::error_code
   writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
+  std::error_code writeSummary();
   std::error_code writeNameIdx(StringRef FName);
   std::error_code writeBody(StringRef FName, const FunctionSamples &S);
 

lib/ProfileData/InstrProfReader.cpp

@@ -595,9 +595,8 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
   } else {
     // For older version of profile data, we need to compute on the fly:
     using namespace IndexedInstrProf;
-    std::vector<uint32_t> Cutoffs(&SummaryCutoffs[0],
-                                  &SummaryCutoffs[NumSummaryCutoffs]);
-    this->Summary = llvm::make_unique<InstrProfSummary>(Cutoffs);
+    this->Summary =
+        llvm::make_unique<InstrProfSummary>(ProfileSummary::DefaultCutoffs);
     this->Summary->computeDetailedSummary();
     return Cur;
   }

lib/ProfileData/InstrProfWriter.cpp

@@ -217,9 +217,7 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
   OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
 
   using namespace IndexedInstrProf;
-  std::vector<uint32_t> Cutoffs(&SummaryCutoffs[0],
-                                &SummaryCutoffs[NumSummaryCutoffs]);
-  InstrProfSummary PS(Cutoffs);
+  InstrProfSummary PS(ProfileSummary::DefaultCutoffs);
   InfoObj->TheProfileSummary = &PS;
 
   // Populate the hash table generator.
@@ -249,7 +247,7 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
   OS.write(0);
 
   // Reserve space to write profile summary data.
-  uint32_t NumEntries = Cutoffs.size();
+  uint32_t NumEntries = ProfileSummary::DefaultCutoffs.size();
   uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
   // Remember the summary offset.
   uint64_t SummaryOffset = OS.tell();

lib/ProfileData/ProfileSummary.cpp

@@ -11,17 +11,36 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ProfileData/ProfileCommon.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/ProfileCommon.h"
+#include "llvm/ProfileData/SampleProf.h"
 
 using namespace llvm;
 
+// A set of cutoff values. Each value, when divided by ProfileSummary::Scale
+// (which is 1000000) is a desired percentile of total counts.
+const std::vector<uint32_t> ProfileSummary::DefaultCutoffs(
+    {10000,  /*  1% */
+     100000, /* 10% */
+     200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, 800000,
+     900000, 950000, 990000, 999000, 999900, 999990, 999999});
+
 void InstrProfSummary::addRecord(const InstrProfRecord &R) {
   addEntryCount(R.Counts[0]);
   for (size_t I = 1, E = R.Counts.size(); I < E; ++I)
     addInternalCount(R.Counts[I]);
 }
 
+// To compute the detailed summary, we consider each line containing samples as
+// equivalent to a block with a count in the instrumented profile.
+void SampleProfileSummary::addRecord(const sampleprof::FunctionSamples &FS) {
+  NumFunctions++;
+  if (FS.getHeadSamples() > MaxHeadSamples)
+    MaxHeadSamples = FS.getHeadSamples();
+  for (const auto &I : FS.getBodySamples())
+    addCount(I.second.getSamples());
+}
+
 // The argument to this method is a vector of cutoff percentages and the return
 // value is a vector of (Cutoff, MinCount, NumCounts) triplets.
 void ProfileSummary::computeDetailedSummary() {

lib/ProfileData/SampleProfReader.cpp

@@ -22,6 +22,7 @@
 
 #include "llvm/ProfileData/SampleProfReader.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorOr.h"
@@ -220,6 +221,8 @@ std::error_code SampleProfileReaderText::read() {
       }
     }
   }
+  if (Result == sampleprof_error::success)
+    computeSummary();
 
   return Result;
 }
@@ -400,6 +403,9 @@ std::error_code SampleProfileReaderBinary::readHeader() {
   else if (*Version != SPVersion())
     return sampleprof_error::unsupported_version;
 
+  if (std::error_code EC = readSummary())
+    return EC;
+
   // Read the name table.
   auto Size = readNumber<uint32_t>();
   if (std::error_code EC = Size.getError())
@@ -415,6 +421,62 @@ std::error_code SampleProfileReaderBinary::readHeader() {
   return sampleprof_error::success;
 }
 
+std::error_code SampleProfileReaderBinary::readSummaryEntry(
+    std::vector<ProfileSummaryEntry> &Entries) {
+  auto Cutoff = readNumber<uint64_t>();
+  if (std::error_code EC = Cutoff.getError())
+    return EC;
+
+  auto MinBlockCount = readNumber<uint64_t>();
+  if (std::error_code EC = MinBlockCount.getError())
+    return EC;
+
+  auto NumBlocks = readNumber<uint64_t>();
+  if (std::error_code EC = NumBlocks.getError())
+    return EC;
+
+  Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
+  return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderBinary::readSummary() {
+  auto TotalCount = readNumber<uint64_t>();
+  if (std::error_code EC = TotalCount.getError())
+    return EC;
+
+  auto MaxBlockCount = readNumber<uint64_t>();
+  if (std::error_code EC = MaxBlockCount.getError())
+    return EC;
+
+  auto MaxFunctionCount = readNumber<uint64_t>();
+  if (std::error_code EC = MaxFunctionCount.getError())
+    return EC;
+
+  auto NumBlocks = readNumber<uint64_t>();
+  if (std::error_code EC = NumBlocks.getError())
+    return EC;
+
+  auto NumFunctions = readNumber<uint64_t>();
+  if (std::error_code EC = NumFunctions.getError())
+    return EC;
+
+  auto NumSummaryEntries = readNumber<uint64_t>();
+  if (std::error_code EC = NumSummaryEntries.getError())
+    return EC;
+
+  std::vector<ProfileSummaryEntry> Entries;
+  for (unsigned i = 0; i < *NumSummaryEntries; i++) {
+    std::error_code EC = readSummaryEntry(Entries);
+    if (EC != sampleprof_error::success)
+      return EC;
+  }
+  Summary = llvm::make_unique<SampleProfileSummary>(
+      *TotalCount, *MaxBlockCount, *MaxFunctionCount, *NumBlocks, *NumFunctions,
+      Entries);
+
+  return sampleprof_error::success;
+}
+
 bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) {
   const uint8_t *Data =
       reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
@@ -518,6 +580,7 @@ std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
     if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
       return EC;
 
+  computeSummary();
   return sampleprof_error::success;
 }
 
@@ -725,3 +788,14 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
 
   return std::move(Reader);
 }
+
+// For text and GCC file formats, we compute the summary after reading the
+// profile. Binary format has the profile summary in its header.
+void SampleProfileReader::computeSummary() {
+  Summary.reset(new SampleProfileSummary(ProfileSummary::DefaultCutoffs));
+  for (const auto &I : Profiles) {
+    const FunctionSamples &Profile = I.second;
+    Summary->addRecord(Profile);
+  }
+  Summary->computeDetailedSummary();
+}

lib/ProfileData/SampleProfWriter.cpp

@@ -120,6 +120,10 @@ std::error_code SampleProfileWriterBinary::writeHeader(
   encodeULEB128(SPMagic(), OS);
   encodeULEB128(SPVersion(), OS);
 
+  computeSummary(ProfileMap);
+  if (auto EC = writeSummary())
+    return EC;
+
   // Generate the name table for all the functions referenced in the profile.
   for (const auto &I : ProfileMap) {
     addName(I.first());
@@ -132,10 +136,25 @@ std::error_code SampleProfileWriterBinary::writeHeader(
     OS << N.first;
     encodeULEB128(0, OS);
   }
-
   return sampleprof_error::success;
 }
 
+std::error_code SampleProfileWriterBinary::writeSummary() {
+  auto &OS = *OutputStream;
+  encodeULEB128(Summary->getTotalSamples(), OS);
+  encodeULEB128(Summary->getMaxSamplesPerLine(), OS);
+  encodeULEB128(Summary->getMaxHeadSamples(), OS);
+  encodeULEB128(Summary->getNumLinesWithSamples(), OS);
+  encodeULEB128(Summary->getNumFunctions(), OS);
+  std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
+  encodeULEB128(Entries.size(), OS);
+  for (auto Entry : Entries) {
+    encodeULEB128(Entry.Cutoff, OS);
+    encodeULEB128(Entry.MinCount, OS);
+    encodeULEB128(Entry.NumCounts, OS);
+  }
+  return sampleprof_error::success;
+}
 std::error_code SampleProfileWriterBinary::writeBody(StringRef FName,
                                                      const FunctionSamples &S) {
   auto &OS = *OutputStream;
@@ -238,3 +257,13 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
 
   return std::move(Writer);
 }
+
+void SampleProfileWriter::computeSummary(
+    const StringMap<FunctionSamples> &ProfileMap) {
+  Summary.reset(new SampleProfileSummary(ProfileSummary::DefaultCutoffs));
+  for (const auto &I : ProfileMap) {
+    const FunctionSamples &Profile = I.second;
+    Summary->addRecord(Profile);
+  }
+  Summary->computeDetailedSummary();
+}

unittests/ProfileData/SampleProfTest.cpp

@@ -55,6 +55,10 @@ struct SampleProfTest : ::testing::Test {
     FooSamples.addTotalSamples(7711);
     FooSamples.addHeadSamples(610);
     FooSamples.addBodySamples(1, 0, 610);
+    FooSamples.addBodySamples(2, 0, 600);
+    FooSamples.addBodySamples(4, 0, 60000);
+    FooSamples.addBodySamples(8, 0, 60351);
+    FooSamples.addBodySamples(10, 0, 605);
 
     StringRef BarName("_Z3bari");
     FunctionSamples BarSamples;
@@ -88,6 +92,32 @@ struct SampleProfTest : ::testing::Test {
     FunctionSamples &ReadBarSamples = ReadProfiles[BarName];
     ASSERT_EQ(20301u, ReadBarSamples.getTotalSamples());
     ASSERT_EQ(1437u, ReadBarSamples.getHeadSamples());
+
+    SampleProfileSummary &Summary = Reader->getSummary();
+    ASSERT_EQ(123603u, Summary.getTotalSamples());
+    ASSERT_EQ(6u, Summary.getNumLinesWithSamples());
+    ASSERT_EQ(2u, Summary.getNumFunctions());
+    ASSERT_EQ(1437u, Summary.getMaxHeadSamples());
+    ASSERT_EQ(60351u, Summary.getMaxSamplesPerLine());
+
+    std::vector<ProfileSummaryEntry> &Details = Summary.getDetailedSummary();
+    uint32_t Cutoff = 800000;
+    auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
+      return PE.Cutoff == Cutoff;
+    };
+    auto EightyPerc = std::find_if(Details.begin(), Details.end(), Predicate);
+    Cutoff = 900000;
+    auto NinetyPerc = std::find_if(Details.begin(), Details.end(), Predicate);
+    Cutoff = 950000;
+    auto NinetyFivePerc =
+        std::find_if(Details.begin(), Details.end(), Predicate);
+    Cutoff = 990000;
+    auto NinetyNinePerc =
+        std::find_if(Details.begin(), Details.end(), Predicate);
+    ASSERT_EQ(60000u, EightyPerc->MinCount);
+    ASSERT_EQ(60000u, NinetyPerc->MinCount);
+    ASSERT_EQ(60000u, NinetyFivePerc->MinCount);
+    ASSERT_EQ(610u, NinetyNinePerc->MinCount);
   }
 };