[MCA] Notify event listeners when instructions transition to the Pending state. NFCI

llvm-svn: 359983

include/llvm/MCA/HWEventListener.h

@@ -39,6 +39,7 @@ public:
     // Events generated by the Retire Control Unit.
     Retired,
     // Events generated by the Scheduler.
+    Pending,
     Ready,
     Issued,
     Executed,

include/llvm/MCA/HardwareUnits/Scheduler.h

@@ -149,8 +149,9 @@ class Scheduler : public HardwareUnit {
   bool promoteToReadySet(SmallVectorImpl<InstRef> &Ready);
 
   // Try to promote instructions from the WaitSet to the PendingSet.
+  // Add promoted instructions to the 'Pending' vector in input.
   // Returns true if at least one instruction was promoted.
-  bool promoteToPendingSet();
+  bool promoteToPendingSet(SmallVectorImpl<InstRef> &Pending);
 
 public:
   Scheduler(const MCSchedModel &Model, LSUnit &Lsu)
@@ -198,6 +199,7 @@ public:
   void issueInstruction(
       InstRef &IR,
       SmallVectorImpl<std::pair<ResourceRef, ResourceCycles>> &Used,
+      SmallVectorImpl<InstRef> &Pending,
       SmallVectorImpl<InstRef> &Ready);
 
   /// Returns true if IR has to be issued immediately, or if IR is a zero
@@ -211,9 +213,15 @@ public:
   /// have changed in state, and that are now available to new instructions.
   /// Instructions executed are added to vector Executed, while vector Ready is
   /// populated with instructions that have become ready in this new cycle.
+  /// Vector Pending is popluated by instructions that have transitioned through
+  /// the pending stat during this cycle. The Pending and Ready sets may not be
+  /// disjoint. An instruction is allowed to transition from the WAIT state to
+  /// the READY state (going through the PENDING state) within a single cycle.
+  /// That means, instructions may appear in both the Pending and Ready set.
   void cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
-                  SmallVectorImpl<InstRef> &Ready,
-                  SmallVectorImpl<InstRef> &Executed);
+                  SmallVectorImpl<InstRef> &Executed,
+                  SmallVectorImpl<InstRef> &Pending,
+                  SmallVectorImpl<InstRef> &Ready);
 
   /// Convert a resource mask into a valid llvm processor resource identifier.
   unsigned getResourceID(uint64_t Mask) const {

include/llvm/MCA/Stages/ExecuteStage.h

@@ -76,6 +76,7 @@ public:
       const InstRef &IR,
       MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const;
   void notifyInstructionExecuted(const InstRef &IR) const;
+  void notifyInstructionPending(const InstRef &IR) const;
   void notifyInstructionReady(const InstRef &IR) const;
   void notifyResourceAvailable(const ResourceRef &RR) const;
 

lib/MCA/HardwareUnits/Scheduler.cpp

@@ -92,6 +92,7 @@ void Scheduler::issueInstructionImpl(
 void Scheduler::issueInstruction(
     InstRef &IR,
     SmallVectorImpl<std::pair<ResourceRef, ResourceCycles>> &UsedResources,
+    SmallVectorImpl<InstRef> &PendingInstructions,
     SmallVectorImpl<InstRef> &ReadyInstructions) {
   const Instruction &Inst = *IR.getInstruction();
   bool HasDependentUsers = Inst.hasDependentUsers();
@@ -102,7 +103,7 @@ void Scheduler::issueInstruction(
   // other dependent instructions. Dependent instructions may be issued during
   // this same cycle if operands have ReadAdvance entries.  Promote those
   // instructions to the ReadySet and notify the caller that those are ready.
-  if (HasDependentUsers && promoteToPendingSet())
+  if (HasDependentUsers && promoteToPendingSet(PendingInstructions))
     promoteToReadySet(ReadyInstructions);
 }
 
@@ -147,7 +148,7 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
   return PromotedElements;
 }
 
-bool Scheduler::promoteToPendingSet() {
+bool Scheduler::promoteToPendingSet(SmallVectorImpl<InstRef> &Pending) {
   // Scan the set of waiting instructions and promote them to the
   // pending set if operands are all ready.
   unsigned RemovedElements = 0;
@@ -166,6 +167,7 @@ bool Scheduler::promoteToPendingSet() {
     LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR
                       << " promoted to the PENDING set.\n");
 
+    Pending.emplace_back(IR);
     PendingSet.emplace_back(IR);
 
     IR.invalidate();
@@ -251,6 +253,7 @@ void Scheduler::analyzeDataDependencies(SmallVectorImpl<InstRef> &RegDeps,
 
 void Scheduler::cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
                            SmallVectorImpl<InstRef> &Executed,
+                           SmallVectorImpl<InstRef> &Pending,
                            SmallVectorImpl<InstRef> &Ready) {
   // Release consumed resources.
   Resources->cycleEvent(Freed);
@@ -265,7 +268,7 @@ void Scheduler::cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
   for (InstRef &IR : WaitSet)
     IR.getInstruction()->cycleEvent();
 
-  promoteToPendingSet();
+  promoteToPendingSet(Pending);
   promoteToReadySet(Ready);
 
   NumDispatchedToThePendingSet = 0;
@@ -299,6 +302,8 @@ bool Scheduler::dispatch(const InstRef &IR) {
     return false;
   }
 
+  // Memory operations that are not in a ready state are initially assigned to
+  // the WaitSet. 
   if (!IS.isReady() ||
       (IS.isMemOp() && LSU.isReady(IR) != IR.getSourceIndex())) {
     LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n");

lib/MCA/Stages/ExecuteStage.cpp

@@ -52,8 +52,10 @@ bool ExecuteStage::isAvailable(const InstRef &IR) const {
 
 Error ExecuteStage::issueInstruction(InstRef &IR) {
   SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> Used;
+  SmallVector<InstRef, 4> Pending;
   SmallVector<InstRef, 4> Ready;
-  HWS.issueInstruction(IR, Used, Ready);
+
+  HWS.issueInstruction(IR, Used, Pending, Ready);
   NumIssuedOpcodes += IR.getInstruction()->getDesc().NumMicroOps;
 
   notifyReservedOrReleasedBuffers(IR, /* Reserved */ false);
@@ -66,6 +68,9 @@ Error ExecuteStage::issueInstruction(InstRef &IR) {
       return S;
   }
 
+  for (const InstRef &I : Pending)
+    notifyInstructionPending(I);
+
   for (const InstRef &I : Ready)
     notifyInstructionReady(I);
   return ErrorSuccess();
@@ -87,9 +92,10 @@ Error ExecuteStage::issueReadyInstructions() {
 Error ExecuteStage::cycleStart() {
   SmallVector<ResourceRef, 8> Freed;
   SmallVector<InstRef, 4> Executed;
+  SmallVector<InstRef, 4> Pending;
   SmallVector<InstRef, 4> Ready;
 
-  HWS.cycleEvent(Freed, Executed, Ready);
+  HWS.cycleEvent(Freed, Executed, Pending, Ready);
   NumDispatchedOpcodes = 0;
   NumIssuedOpcodes = 0;
 
@@ -103,6 +109,9 @@ Error ExecuteStage::cycleStart() {
       return S;
   }
 
+  for (const InstRef &IR : Pending)
+    notifyInstructionPending(IR);
+
   for (const InstRef &IR : Ready)
     notifyInstructionReady(IR);
 
@@ -126,7 +135,6 @@ Error ExecuteStage::cycleEnd() {
                       << format_hex(Mask, 16) << '\n');
     HWPressureEvent Ev(HWPressureEvent::RESOURCES, Insts, Mask);
     notifyEvent(Ev);
-    return ErrorSuccess();
   }
 
   SmallVector<InstRef, 8> RegDeps;
@@ -165,6 +173,7 @@ Error ExecuteStage::handleInstructionEliminated(InstRef &IR) {
 #ifndef NDEBUG
   verifyInstructionEliminated(IR);
 #endif
+  notifyInstructionPending(IR);
   notifyInstructionReady(IR);
   notifyInstructionIssued(IR, {});
   IR.getInstruction()->forceExecuted();
@@ -189,10 +198,17 @@ Error ExecuteStage::execute(InstRef &IR) {
   // be released after MCIS is issued, and all the ResourceCycles for those
   // units have been consumed.
   bool IsReadyInstruction = HWS.dispatch(IR);
-  NumDispatchedOpcodes += IR.getInstruction()->getDesc().NumMicroOps;
+  const Instruction &Inst = *IR.getInstruction();
+  NumDispatchedOpcodes += Inst.getDesc().NumMicroOps;
   notifyReservedOrReleasedBuffers(IR, /* Reserved */ true);
-  if (!IsReadyInstruction)
+ 
+  if (!IsReadyInstruction) {
+    if (Inst.isPending())
+      notifyInstructionPending(IR);
     return ErrorSuccess();
+  }
+
+  notifyInstructionPending(IR);
 
   // If we did not return early, then the scheduler is ready for execution.
   notifyInstructionReady(IR);
@@ -212,6 +228,12 @@ void ExecuteStage::notifyInstructionExecuted(const InstRef &IR) const {
       HWInstructionEvent(HWInstructionEvent::Executed, IR));
 }
 
+void ExecuteStage::notifyInstructionPending(const InstRef &IR) const {
+  LLVM_DEBUG(dbgs() << "[E] Instruction Pending: #" << IR << '\n');
+  notifyEvent<HWInstructionEvent>(
+      HWInstructionEvent(HWInstructionEvent::Pending, IR));
+}
+
 void ExecuteStage::notifyInstructionReady(const InstRef &IR) const {
   LLVM_DEBUG(dbgs() << "[E] Instruction Ready: #" << IR << '\n');
   notifyEvent<HWInstructionEvent>(