Bug 1209515 part 1 - IonBuilder: Attach hit counts on the MIRGraph. r=bhackett

js/src/jit/IonBuilder.cpp

@@ -7280,6 +7280,8 @@ IonBuilder::addBlock(MBasicBlock* block, uint32_t loopDepth)
 {
     if (!block)
         return nullptr;
+    if (block->pc() && script()->hasScriptCounts())
+        block->setHitCount(script()->getHitCount(block->pc()));
     graph().addBlock(block);
     block->setLoopDepth(loopDepth);
     return block;
@@ -7316,6 +7318,7 @@ IonBuilder::newBlockAfter(MBasicBlock* at, MBasicBlock* predecessor, jsbytecode*
                                           bytecodeSite(pc), MBasicBlock::NORMAL);
     if (!block)
         return nullptr;
+    block->setHitCount(0); // osr block
     graph().insertBlockAfter(at, block);
     return block;
 }

js/src/jit/JSONSpewer.cpp

@@ -259,6 +259,8 @@ JSONSpewer::spewMIR(MIRGraph* mir)
         beginObject();
 
         integerProperty("number", block->id());
+        if (block->getHitState() == MBasicBlock::HitState::Count)
+            integerProperty("count", block->getHitCount());
 
         beginListProperty("attributes");
         if (block->isLoopBackedge())

js/src/jit/JitOptions.cpp

@@ -98,6 +98,9 @@ JitOptions::JitOptions()
     // Toggles whether Loop Unrolling is globally disabled.
     SET_DEFAULT(disableLoopUnrolling, true);
 
+    // Toggle whether Profile Guided Optimization is globally disabled.
+    SET_DEFAULT(disablePgo, true);
+
     // Toggles whether instruction reordering is globally disabled.
     SET_DEFAULT(disableInstructionReordering, false);
 

js/src/jit/JitOptions.h

@@ -55,6 +55,7 @@ struct JitOptions
     bool disableInlining;
     bool disableLicm;
     bool disableLoopUnrolling;
+    bool disablePgo;
     bool disableInstructionReordering;
     bool disableRangeAnalysis;
     bool disableScalarReplacement;

js/src/jit/MIRGraph.cpp

@@ -404,7 +404,9 @@ MBasicBlock::MBasicBlock(MIRGraph& graph, CompileInfo& info, BytecodeSite* site,
     mark_(false),
     immediatelyDominated_(graph.alloc()),
     immediateDominator_(nullptr),
-    trackedSite_(site)
+    trackedSite_(site),
+    hitCount_(0),
+    hitState_(HitState::NotDefined)
 #if defined (JS_ION_PERF)
     , lineno_(0u),
     columnIndex_(0u)

js/src/jit/MIRGraph.h

@@ -603,6 +603,32 @@ class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock>
     void dump(GenericPrinter& out);
     void dump();
 
+    // Hit count
+    enum class HitState {
+        // Not hit information is attached to this basic block.
+        NotDefined,
+
+        // The hit information is a raw counter. Note that due to inlining this
+        // counter is not guaranteed to be consistent over the graph.
+        Count,
+
+        // The hit information is a frequency, which is a form of normalized
+        // counter, where a hit-count can be compared against any previous block
+        // in the graph.
+        Frequency
+    };
+    HitState getHitState() const {
+        return hitState_;
+    }
+    void setHitCount(uint64_t count) {
+        hitCount_ = count;
+        hitState_ = HitState::Count;
+    }
+    uint64_t getHitCount() const {
+        MOZ_ASSERT(hitState_ == HitState::Count);
+        return hitCount_;
+    }
+
     // Track bailouts by storing the current pc in MIR instruction added at
     // this cycle. This is also used for tracking calls and optimizations when
     // profiling.
@@ -666,6 +692,11 @@ class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock>
 
     BytecodeSite* trackedSite_;
 
+    // Record the number of times a block got visited. Note, due to inlined
+    // scripts these numbers might not be continuous.
+    uint64_t hitCount_;
+    HitState hitState_;
+
 #if defined(JS_ION_PERF) || defined(DEBUG)
     unsigned lineno_;
     unsigned columnIndex_;

js/src/jscompartment.cpp

@@ -18,6 +18,7 @@
 
 #include "gc/Marking.h"
 #include "jit/JitCompartment.h"
+#include "jit/JitOptions.h"
 #include "js/Date.h"
 #include "js/Proxy.h"
 #include "js/RootingAPI.h"
@@ -1075,6 +1076,15 @@ JSCompartment::updateDebuggerObservesCoverage()
     clearScriptCounts();
 }
 
+bool
+JSCompartment::collectCoverage() const
+{
+    return !js_JitOptions.disablePgo ||
+           debuggerObservesCoverage() ||
+           runtimeFromAnyThread()->profilingScripts ||
+           runtimeFromAnyThread()->lcovOutput.isEnabled();
+}
+
 void
 JSCompartment::clearScriptCounts()
 {

js/src/jscompartment.h

@@ -680,11 +680,7 @@ struct JSCompartment
 
     // The code coverage can be enabled either for each compartment, with the
     // Debugger API, or for the entire runtime.
-    bool collectCoverage() const {
-        return debuggerObservesCoverage() ||
-               runtimeFromAnyThread()->profilingScripts ||
-               runtimeFromAnyThread()->lcovOutput.isEnabled();
-    }
+    bool collectCoverage() const;
     void clearScriptCounts();
 
     bool needsDelazificationForDebugger() const {

js/src/jsscript.cpp

@@ -1415,6 +1415,20 @@ ScriptCounts::maybeGetPCCounts(size_t offset) const {
     return elem;
 }
 
+const js::PCCounts*
+ScriptCounts::getImmediatePrecedingPCCounts(size_t offset) const
+{
+    PCCounts searched = PCCounts(offset);
+    const PCCounts* elem = std::lower_bound(pcCounts_.begin(), pcCounts_.end(), searched);
+    if (elem == pcCounts_.end())
+        return &pcCounts_.back();
+    if (elem->pcOffset() == offset)
+        return elem;
+    if (elem != pcCounts_.begin())
+        return elem - 1;
+    return nullptr;
+}
+
 const js::PCCounts*
 ScriptCounts::maybeGetThrowCounts(size_t offset) const {
     PCCounts searched = PCCounts(offset);
@@ -1424,6 +1438,23 @@ ScriptCounts::maybeGetThrowCounts(size_t offset) const {
     return elem;
 }
 
+const js::PCCounts*
+ScriptCounts::getImmediatePrecedingThrowCounts(size_t offset) const
+{
+    PCCounts searched = PCCounts(offset);
+    const PCCounts* elem = std::lower_bound(throwCounts_.begin(), throwCounts_.end(), searched);
+    if (elem == throwCounts_.end()) {
+        if (throwCounts_.begin() == throwCounts_.end())
+            return nullptr;
+        return &throwCounts_.back();
+    }
+    if (elem->pcOffset() == offset)
+        return elem;
+    if (elem != throwCounts_.begin())
+        return elem - 1;
+    return nullptr;
+}
+
 js::PCCounts*
 ScriptCounts::getThrowCounts(size_t offset) {
     PCCounts searched = PCCounts(offset);
@@ -1462,6 +1493,33 @@ JSScript::getThrowCounts(jsbytecode* pc) {
     return getScriptCounts().getThrowCounts(pcToOffset(pc));
 }
 
+uint64_t
+JSScript::getHitCount(jsbytecode* pc)
+{
+    MOZ_ASSERT(containsPC(pc));
+    if (pc < main())
+        pc = main();
+
+    ScriptCounts& sc = getScriptCounts();
+    size_t targetOffset = pcToOffset(pc);
+    const js::PCCounts* baseCount = sc.getImmediatePrecedingPCCounts(targetOffset);
+    if (!baseCount)
+        return 0;
+    if (baseCount->pcOffset() == targetOffset)
+        return baseCount->numExec();
+    MOZ_ASSERT(baseCount->pcOffset() < targetOffset);
+    uint64_t count = baseCount->numExec();
+    do {
+        const js::PCCounts* throwCount = sc.getImmediatePrecedingThrowCounts(targetOffset);
+        if (!throwCount)
+            return count;
+        if (throwCount->pcOffset() <= baseCount->pcOffset())
+            return count;
+        count -= throwCount->numExec();
+        targetOffset = throwCount->pcOffset() - 1;
+    } while (true);
+}
+
 void
 JSScript::addIonCounts(jit::IonScriptCounts* ionCounts)
 {

js/src/jsscript.h

@@ -468,10 +468,22 @@ class ScriptCounts
     PCCounts* maybeGetPCCounts(size_t offset);
     const PCCounts* maybeGetPCCounts(size_t offset) const;
 
+    // PCCounts are stored at jump-target offsets. This function looks for the
+    // previous PCCount which is in the same basic block as the current offset.
+    const PCCounts* getImmediatePrecedingPCCounts(size_t offset) const;
+
     // Return the counter used to count the number of throws. Returns null if
     // the element is not found.
     const PCCounts* maybeGetThrowCounts(size_t offset) const;
 
+    // Throw counts are stored at the location of each throwing
+    // instruction. This function looks for the previous throw count.
+    //
+    // Note: if the offset of the returned count is higher than the offset of
+    // the immediate preceding PCCount, then this throw happened in the same
+    // basic block.
+    const PCCounts* getImmediatePrecedingThrowCounts(size_t offset) const;
+
     // Return the counter used to count the number of throws. Allocate it if
     // none exists yet. Returns null if the allocation failed.
     PCCounts* getThrowCounts(size_t offset);
@@ -1671,6 +1683,7 @@ class JSScript : public js::gc::TenuredCell
     js::PCCounts* maybeGetPCCounts(jsbytecode* pc);
     const js::PCCounts* maybeGetThrowCounts(jsbytecode* pc);
     js::PCCounts* getThrowCounts(jsbytecode* pc);
+    uint64_t getHitCount(jsbytecode* pc);
     void addIonCounts(js::jit::IonScriptCounts* ionCounts);
     js::jit::IonScriptCounts* getIonCounts();
     void releaseScriptCounts(js::ScriptCounts* counts);

js/src/shell/js.cpp

@@ -6245,6 +6245,15 @@ SetRuntimeOptions(JSRuntime* rt, const OptionParser& op)
             return OptionFailure("ion-edgecase-analysis", str);
     }
 
+    if (const char* str = op.getStringOption("ion-pgo")) {
+        if (strcmp(str, "on") == 0)
+            jit::js_JitOptions.disablePgo = false;
+        else if (strcmp(str, "off") == 0)
+            jit::js_JitOptions.disablePgo = true;
+        else
+            return OptionFailure("ion-pgo", str);
+    }
+
     if (const char* str = op.getStringOption("ion-range-analysis")) {
         if (strcmp(str, "on") == 0)
             jit::js_JitOptions.disableRangeAnalysis = false;
@@ -6599,6 +6608,8 @@ main(int argc, char** argv, char** envp)
                                "Loop invariant code motion (default: on, off to disable)")
         || !op.addStringOption('\0', "ion-edgecase-analysis", "on/off",
                                "Find edge cases where Ion can avoid bailouts (default: on, off to disable)")
+        || !op.addStringOption('\0', "ion-pgo", "on/off",
+                               "Profile guided optimization (default: off, on to enable)")
         || !op.addStringOption('\0', "ion-range-analysis", "on/off",
                                "Range analysis (default: on, off to disable)")
 #if defined(__APPLE__)