[PGO][PGSO] ProfileSummary changes.

(Split of off D67120)

ProfileSummary changes for profile guided size optimization.

Differential Revision: https://reviews.llvm.org/D67377

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@372783 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/Analysis/ProfileSummaryInfo.h

@@ -52,6 +52,15 @@ private:
   // because the number of profile counts required to reach the hot
   // percentile is above a huge threshold.
   Optional<bool> HasHugeWorkingSetSize;
+  // True if the working set size of the code is considered large,
+  // because the number of profile counts required to reach the hot
+  // percentile is above a large threshold.
+  Optional<bool> HasLargeWorkingSetSize;
+  // Compute the threshold for a given cutoff.
+  Optional<uint64_t> computeThreshold(int PercentileCutoff);
+  // The map that caches the threshold values. The keys are the percentile
+  // cutoff values and the values are the corresponding threshold values.
+  DenseMap<int, uint64_t> ThresholdCache;
 
 public:
   ProfileSummaryInfo(Module &M) : M(M) {}
@@ -96,6 +105,8 @@ public:
                                      bool AllowSynthetic = false);
   /// Returns true if the working set size of the code is considered huge.
   bool hasHugeWorkingSetSize();
+  /// Returns true if the working set size of the code is considered large.
+  bool hasLargeWorkingSetSize();
   /// Returns true if \p F has hot function entry.
   bool isFunctionEntryHot(const Function *F);
   /// Returns true if \p F contains hot code.
@@ -104,14 +115,26 @@ public:
   bool isFunctionEntryCold(const Function *F);
   /// Returns true if \p F contains only cold code.
   bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI);
+  /// Returns true if \p F contains hot code with regard to a given hot
+  /// percentile cutoff value.
+  bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff,
+                                             const Function *F,
+                                             BlockFrequencyInfo &BFI);
   /// Returns true if count \p C is considered hot.
   bool isHotCount(uint64_t C);
   /// Returns true if count \p C is considered cold.
   bool isColdCount(uint64_t C);
+  /// Returns true if count \p C is considered hot with regard to a given
+  /// hot percentile cutoff value.
+  bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C);
   /// Returns true if BasicBlock \p BB is considered hot.
   bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI);
   /// Returns true if BasicBlock \p BB is considered cold.
   bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI);
+  /// Returns true if BasicBlock \p BB is considered hot with regard to a given
+  /// hot percentile cutoff value.
+  bool isHotBlockNthPercentile(int PercentileCutoff,
+                               const BasicBlock *BB, BlockFrequencyInfo *BFI);
   /// Returns true if CallSite \p CS is considered hot.
   bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI);
   /// Returns true if Callsite \p CS is considered cold.

lib/Analysis/ProfileSummaryInfo.cpp

@@ -45,6 +45,13 @@ static cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold(
              " blocks required to reach the -profile-summary-cutoff-hot"
              " percentile exceeds this count."));
 
+static cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold(
+    "profile-summary-large-working-set-size-threshold", cl::Hidden,
+    cl::init(12500), cl::ZeroOrMore,
+    cl::desc("The code working set size is considered large if the number of"
+             " blocks required to reach the -profile-summary-cutoff-hot"
+             " percentile exceeds this count."));
+
 // The next two options override the counts derived from summary computation and
 // are useful for debugging purposes.
 static cl::opt<int> ProfileSummaryHotCount(
@@ -186,6 +193,31 @@ bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F,
   return true;
 }
 
+// Like isFunctionHotInCallGraph but for a given cutoff.
+bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile(
+    int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) {
+  if (!F || !computeSummary())
+    return false;
+  if (auto FunctionCount = F->getEntryCount())
+    if (isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount()))
+      return true;
+
+  if (hasSampleProfile()) {
+    uint64_t TotalCallCount = 0;
+    for (const auto &BB : *F)
+      for (const auto &I : BB)
+        if (isa<CallInst>(I) || isa<InvokeInst>(I))
+          if (auto CallCount = getProfileCount(&I, nullptr))
+            TotalCallCount += CallCount.getValue();
+    if (isHotCountNthPercentile(PercentileCutoff, TotalCallCount))
+      return true;
+  }
+  for (const auto &BB : *F)
+    if (isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI))
+      return true;
+  return false;
+}
+
 /// Returns true if the function's entry is a cold. If it returns false, it
 /// either means it is not cold or it is unknown whether it is cold or not (for
 /// example, no profile data is available).
@@ -222,6 +254,23 @@ void ProfileSummaryInfo::computeThresholds() {
          "Cold count threshold cannot exceed hot count threshold!");
   HasHugeWorkingSetSize =
       HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
+  HasLargeWorkingSetSize =
+      HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
+}
+
+Optional<uint64_t> ProfileSummaryInfo::computeThreshold(int PercentileCutoff) {
+  if (!computeSummary())
+    return None;
+  auto iter = ThresholdCache.find(PercentileCutoff);
+  if (iter != ThresholdCache.end()) {
+    return iter->second;
+  }
+  auto &DetailedSummary = Summary->getDetailedSummary();
+  auto &Entry =
+      getEntryForPercentile(DetailedSummary, PercentileCutoff);
+  uint64_t CountThreshold = Entry.MinCount;
+  ThresholdCache[PercentileCutoff] = CountThreshold;
+  return CountThreshold;
 }
 
 bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
@@ -230,6 +279,12 @@ bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
   return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue();
 }
 
+bool ProfileSummaryInfo::hasLargeWorkingSetSize() {
+  if (!HasLargeWorkingSetSize)
+    computeThresholds();
+  return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue();
+}
+
 bool ProfileSummaryInfo::isHotCount(uint64_t C) {
   if (!HotCountThreshold)
     computeThresholds();
@@ -242,6 +297,11 @@ bool ProfileSummaryInfo::isColdCount(uint64_t C) {
   return ColdCountThreshold && C <= ColdCountThreshold.getValue();
 }
 
+bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) {
+  auto CountThreshold = computeThreshold(PercentileCutoff);
+  return CountThreshold && C >= CountThreshold.getValue();
+}
+
 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() {
   if (!HotCountThreshold)
     computeThresholds();
@@ -265,6 +325,13 @@ bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB,
   return Count && isColdCount(*Count);
 }
 
+bool ProfileSummaryInfo::isHotBlockNthPercentile(int PercentileCutoff,
+                                                 const BasicBlock *BB,
+                                                 BlockFrequencyInfo *BFI) {
+  auto Count = BFI->getBlockProfileCount(BB);
+  return Count && isHotCountNthPercentile(PercentileCutoff, *Count);
+}
+
 bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS,
                                        BlockFrequencyInfo *BFI) {
   auto C = getProfileCount(CS.getInstruction(), BFI);

unittests/Analysis/ProfileSummaryInfoTest.cpp

@@ -137,6 +137,18 @@ TEST_F(ProfileSummaryInfoTest, TestCommon) {
   EXPECT_FALSE(PSI.isColdCount(100));
   EXPECT_FALSE(PSI.isHotCount(100));
 
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(990000, 400));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 2));
+
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 400));
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(999999, 2));
+
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 400));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 2));
+
   EXPECT_TRUE(PSI.isFunctionEntryHot(F));
   EXPECT_FALSE(PSI.isFunctionEntryHot(G));
   EXPECT_FALSE(PSI.isFunctionEntryHot(H));
@@ -160,6 +172,21 @@ TEST_F(ProfileSummaryInfoTest, InstrProf) {
   EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI));
   EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI));
 
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI));
+
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI));
+
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI));
+
   CallSite CS1(BB1->getFirstNonPHI());
   auto *CI2 = BB2->getFirstNonPHI();
   CallSite CS2(CI2);
@@ -192,6 +219,21 @@ TEST_F(ProfileSummaryInfoTest, SampleProf) {
   EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI));
   EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI));
 
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI));
+
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI));
+
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI));
+
   CallSite CS1(BB1->getFirstNonPHI());
   auto *CI2 = BB2->getFirstNonPHI();
   // Manually attach branch weights metadata to the call instruction.