From c70f11adf61c070f44427b95bdc1cb95c0f09434 Mon Sep 17 00:00:00 2001 From: Andrea Di Biagio Date: Sun, 5 May 2019 16:07:27 +0000 Subject: [PATCH] [MCA] Notify event listeners when instructions transition to the Pending state. NFCI llvm-svn: 359983 --- include/llvm/MCA/HWEventListener.h | 1 + include/llvm/MCA/HardwareUnits/Scheduler.h | 14 ++++++++-- include/llvm/MCA/Stages/ExecuteStage.h | 1 + lib/MCA/HardwareUnits/Scheduler.cpp | 11 ++++++-- lib/MCA/Stages/ExecuteStage.cpp | 32 ++++++++++++++++++---- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/include/llvm/MCA/HWEventListener.h b/include/llvm/MCA/HWEventListener.h index 2f81b874a62..e11d06de2b2 100644 --- a/include/llvm/MCA/HWEventListener.h +++ b/include/llvm/MCA/HWEventListener.h @@ -39,6 +39,7 @@ public: // Events generated by the Retire Control Unit. Retired, // Events generated by the Scheduler. + Pending, Ready, Issued, Executed, diff --git a/include/llvm/MCA/HardwareUnits/Scheduler.h b/include/llvm/MCA/HardwareUnits/Scheduler.h index dc6722c4e85..41d062be568 100644 --- a/include/llvm/MCA/HardwareUnits/Scheduler.h +++ b/include/llvm/MCA/HardwareUnits/Scheduler.h @@ -149,8 +149,9 @@ class Scheduler : public HardwareUnit { bool promoteToReadySet(SmallVectorImpl &Ready); // Try to promote instructions from the WaitSet to the PendingSet. + // Add promoted instructions to the 'Pending' vector in input. // Returns true if at least one instruction was promoted. - bool promoteToPendingSet(); + bool promoteToPendingSet(SmallVectorImpl &Pending); public: Scheduler(const MCSchedModel &Model, LSUnit &Lsu) @@ -198,6 +199,7 @@ public: void issueInstruction( InstRef &IR, SmallVectorImpl> &Used, + SmallVectorImpl &Pending, SmallVectorImpl &Ready); /// Returns true if IR has to be issued immediately, or if IR is a zero @@ -211,9 +213,15 @@ public: /// have changed in state, and that are now available to new instructions. /// Instructions executed are added to vector Executed, while vector Ready is /// populated with instructions that have become ready in this new cycle. + /// Vector Pending is popluated by instructions that have transitioned through + /// the pending stat during this cycle. The Pending and Ready sets may not be + /// disjoint. An instruction is allowed to transition from the WAIT state to + /// the READY state (going through the PENDING state) within a single cycle. + /// That means, instructions may appear in both the Pending and Ready set. void cycleEvent(SmallVectorImpl &Freed, - SmallVectorImpl &Ready, - SmallVectorImpl &Executed); + SmallVectorImpl &Executed, + SmallVectorImpl &Pending, + SmallVectorImpl &Ready); /// Convert a resource mask into a valid llvm processor resource identifier. unsigned getResourceID(uint64_t Mask) const { diff --git a/include/llvm/MCA/Stages/ExecuteStage.h b/include/llvm/MCA/Stages/ExecuteStage.h index c20173b0d9f..03737e0220e 100644 --- a/include/llvm/MCA/Stages/ExecuteStage.h +++ b/include/llvm/MCA/Stages/ExecuteStage.h @@ -76,6 +76,7 @@ public: const InstRef &IR, MutableArrayRef> Used) const; void notifyInstructionExecuted(const InstRef &IR) const; + void notifyInstructionPending(const InstRef &IR) const; void notifyInstructionReady(const InstRef &IR) const; void notifyResourceAvailable(const ResourceRef &RR) const; diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp index 5b2527b886e..9eeea9d0113 100644 --- a/lib/MCA/HardwareUnits/Scheduler.cpp +++ b/lib/MCA/HardwareUnits/Scheduler.cpp @@ -92,6 +92,7 @@ void Scheduler::issueInstructionImpl( void Scheduler::issueInstruction( InstRef &IR, SmallVectorImpl> &UsedResources, + SmallVectorImpl &PendingInstructions, SmallVectorImpl &ReadyInstructions) { const Instruction &Inst = *IR.getInstruction(); bool HasDependentUsers = Inst.hasDependentUsers(); @@ -102,7 +103,7 @@ void Scheduler::issueInstruction( // other dependent instructions. Dependent instructions may be issued during // this same cycle if operands have ReadAdvance entries. Promote those // instructions to the ReadySet and notify the caller that those are ready. - if (HasDependentUsers && promoteToPendingSet()) + if (HasDependentUsers && promoteToPendingSet(PendingInstructions)) promoteToReadySet(ReadyInstructions); } @@ -147,7 +148,7 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { return PromotedElements; } -bool Scheduler::promoteToPendingSet() { +bool Scheduler::promoteToPendingSet(SmallVectorImpl &Pending) { // Scan the set of waiting instructions and promote them to the // pending set if operands are all ready. unsigned RemovedElements = 0; @@ -166,6 +167,7 @@ bool Scheduler::promoteToPendingSet() { LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR << " promoted to the PENDING set.\n"); + Pending.emplace_back(IR); PendingSet.emplace_back(IR); IR.invalidate(); @@ -251,6 +253,7 @@ void Scheduler::analyzeDataDependencies(SmallVectorImpl &RegDeps, void Scheduler::cycleEvent(SmallVectorImpl &Freed, SmallVectorImpl &Executed, + SmallVectorImpl &Pending, SmallVectorImpl &Ready) { // Release consumed resources. Resources->cycleEvent(Freed); @@ -265,7 +268,7 @@ void Scheduler::cycleEvent(SmallVectorImpl &Freed, for (InstRef &IR : WaitSet) IR.getInstruction()->cycleEvent(); - promoteToPendingSet(); + promoteToPendingSet(Pending); promoteToReadySet(Ready); NumDispatchedToThePendingSet = 0; @@ -299,6 +302,8 @@ bool Scheduler::dispatch(const InstRef &IR) { return false; } + // Memory operations that are not in a ready state are initially assigned to + // the WaitSet. if (!IS.isReady() || (IS.isMemOp() && LSU.isReady(IR) != IR.getSourceIndex())) { LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); diff --git a/lib/MCA/Stages/ExecuteStage.cpp b/lib/MCA/Stages/ExecuteStage.cpp index 49210e06cd4..a2b361fcd1b 100644 --- a/lib/MCA/Stages/ExecuteStage.cpp +++ b/lib/MCA/Stages/ExecuteStage.cpp @@ -52,8 +52,10 @@ bool ExecuteStage::isAvailable(const InstRef &IR) const { Error ExecuteStage::issueInstruction(InstRef &IR) { SmallVector, 4> Used; + SmallVector Pending; SmallVector Ready; - HWS.issueInstruction(IR, Used, Ready); + + HWS.issueInstruction(IR, Used, Pending, Ready); NumIssuedOpcodes += IR.getInstruction()->getDesc().NumMicroOps; notifyReservedOrReleasedBuffers(IR, /* Reserved */ false); @@ -66,6 +68,9 @@ Error ExecuteStage::issueInstruction(InstRef &IR) { return S; } + for (const InstRef &I : Pending) + notifyInstructionPending(I); + for (const InstRef &I : Ready) notifyInstructionReady(I); return ErrorSuccess(); @@ -87,9 +92,10 @@ Error ExecuteStage::issueReadyInstructions() { Error ExecuteStage::cycleStart() { SmallVector Freed; SmallVector Executed; + SmallVector Pending; SmallVector Ready; - HWS.cycleEvent(Freed, Executed, Ready); + HWS.cycleEvent(Freed, Executed, Pending, Ready); NumDispatchedOpcodes = 0; NumIssuedOpcodes = 0; @@ -103,6 +109,9 @@ Error ExecuteStage::cycleStart() { return S; } + for (const InstRef &IR : Pending) + notifyInstructionPending(IR); + for (const InstRef &IR : Ready) notifyInstructionReady(IR); @@ -126,7 +135,6 @@ Error ExecuteStage::cycleEnd() { << format_hex(Mask, 16) << '\n'); HWPressureEvent Ev(HWPressureEvent::RESOURCES, Insts, Mask); notifyEvent(Ev); - return ErrorSuccess(); } SmallVector RegDeps; @@ -165,6 +173,7 @@ Error ExecuteStage::handleInstructionEliminated(InstRef &IR) { #ifndef NDEBUG verifyInstructionEliminated(IR); #endif + notifyInstructionPending(IR); notifyInstructionReady(IR); notifyInstructionIssued(IR, {}); IR.getInstruction()->forceExecuted(); @@ -189,10 +198,17 @@ Error ExecuteStage::execute(InstRef &IR) { // be released after MCIS is issued, and all the ResourceCycles for those // units have been consumed. bool IsReadyInstruction = HWS.dispatch(IR); - NumDispatchedOpcodes += IR.getInstruction()->getDesc().NumMicroOps; + const Instruction &Inst = *IR.getInstruction(); + NumDispatchedOpcodes += Inst.getDesc().NumMicroOps; notifyReservedOrReleasedBuffers(IR, /* Reserved */ true); - if (!IsReadyInstruction) + + if (!IsReadyInstruction) { + if (Inst.isPending()) + notifyInstructionPending(IR); return ErrorSuccess(); + } + + notifyInstructionPending(IR); // If we did not return early, then the scheduler is ready for execution. notifyInstructionReady(IR); @@ -212,6 +228,12 @@ void ExecuteStage::notifyInstructionExecuted(const InstRef &IR) const { HWInstructionEvent(HWInstructionEvent::Executed, IR)); } +void ExecuteStage::notifyInstructionPending(const InstRef &IR) const { + LLVM_DEBUG(dbgs() << "[E] Instruction Pending: #" << IR << '\n'); + notifyEvent( + HWInstructionEvent(HWInstructionEvent::Pending, IR)); +} + void ExecuteStage::notifyInstructionReady(const InstRef &IR) const { LLVM_DEBUG(dbgs() << "[E] Instruction Ready: #" << IR << '\n'); notifyEvent(