diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index 8b52b5a9c70..d13ee842578 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -18,6 +18,7 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/TargetSchedule.h" #include "llvm/Support/Compiler.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/ADT/SmallSet.h" @@ -181,6 +182,9 @@ namespace llvm { /// Live Intervals provides reaching defs in preRA scheduling. LiveIntervals *LIS; + /// TargetSchedModel provides an interface to the machine model. + TargetSchedModel SchedModel; + /// isPostRA flag indicates vregs cannot be present. bool IsPostRA; diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index 5023f4906e8..d2a26afe999 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -45,17 +45,33 @@ public: /// Return true if this machine model includes an instruction-level scheduling /// model. This is more detailed than the course grain IssueWidth and default /// latency properties, but separate from the per-cycle itinerary data. - bool hasInstrSchedModel() const { - return SchedModel.hasInstrSchedModel(); - } + bool hasInstrSchedModel() const { return SchedModel.hasInstrSchedModel(); } /// Return true if this machine model includes cycle-to-cycle itinerary /// data. This models scheduling at each stage in the processor pipeline. - bool hasInstrItineraries() const { - return SchedModel.hasInstrItineraries(); - } + bool hasInstrItineraries() const { return !InstrItins.isEmpty(); } + + /// computeOperandLatency - Compute and return the latency of the given data + /// dependent def and use when the operand indices are already known. UseMI + /// may be NULL for an unknown user. + /// + /// FindMin may be set to get the minimum vs. expected latency. Minimum + /// latency is used for scheduling groups, while expected latency is for + /// instruction cost and critical path. + unsigned computeOperandLatency(const MachineInstr *DefMI, unsigned DefOperIdx, + const MachineInstr *UseMI, unsigned UseOperIdx, + bool FindMin) const; unsigned getProcessorID() const { return SchedModel.getProcessorID(); } + +private: + /// getDefLatency is a helper for computeOperandLatency. Return the + /// instruction's latency if operand lookup is not required. + /// Otherwise return -1. + int getDefLatency(const MachineInstr *DefMI, bool FindMin) const; + + /// Return the MCSchedClassDesc for this instruction. + const MCSchedClassDesc *resolveSchedClass(const MachineInstr *MI) const; }; } // namespace llvm diff --git a/include/llvm/MC/MCSchedule.h b/include/llvm/MC/MCSchedule.h index cff175b616a..62fb22fb2aa 100644 --- a/include/llvm/MC/MCSchedule.h +++ b/include/llvm/MC/MCSchedule.h @@ -208,14 +208,7 @@ public: unsigned getProcessorID() const { return ProcID; } /// Does this machine model include instruction-level scheduling. - bool hasInstrSchedModel() const { - return SchedClassTable; - } - - /// Does this machine model include cycle-to-cycle itineraries. - bool hasInstrItineraries() const { - return InstrItineraries; - } + bool hasInstrSchedModel() const { return SchedClassTable != NULL; } const MCProcResourceDesc *getProcResource(unsigned ProcResourceIdx) const { assert(hasInstrSchedModel() && "No scheduling machine model"); diff --git a/include/llvm/Target/TargetInstrInfo.h b/include/llvm/Target/TargetInstrInfo.h index bdf6a6df60f..c5c5a7a9c93 100644 --- a/include/llvm/Target/TargetInstrInfo.h +++ b/include/llvm/Target/TargetInstrInfo.h @@ -824,6 +824,9 @@ public: unsigned defaultDefLatency(const MCSchedModel *SchedModel, const MachineInstr *DefMI) const; + int computeDefOperandLatency(const InstrItineraryData *ItinData, + const MachineInstr *DefMI, bool FindMin) const; + /// isHighLatencyDef - Return true if this opcode has high latency to its /// result. virtual bool isHighLatencyDef(int opc) const { return false; } diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp index 7e7f835040b..8ed66f70443 100644 --- a/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/lib/CodeGen/TargetInstrInfoImpl.cpp @@ -606,13 +606,13 @@ getOperandLatency(const InstrItineraryData *ItinData, /// If we can determine the operand latency from the def only, without itinerary /// lookup, do so. Otherwise return -1. -static int computeDefOperandLatency( - const TargetInstrInfo *TII, const InstrItineraryData *ItinData, - const MachineInstr *DefMI, bool FindMin) { +int TargetInstrInfo::computeDefOperandLatency( + const InstrItineraryData *ItinData, + const MachineInstr *DefMI, bool FindMin) const { // Let the target hook getInstrLatency handle missing itineraries. if (!ItinData) - return TII->getInstrLatency(ItinData, DefMI); + return getInstrLatency(ItinData, DefMI); // Return a latency based on the itinerary properties and defining instruction // if possible. Some common subtargets don't require per-operand latency, @@ -621,7 +621,7 @@ static int computeDefOperandLatency( // If MinLatency is valid, call getInstrLatency. This uses Stage latency if // it exists before defaulting to MinLatency. if (ItinData->SchedModel->MinLatency >= 0) - return TII->getInstrLatency(ItinData, DefMI); + return getInstrLatency(ItinData, DefMI); // If MinLatency is invalid, OperandLatency is interpreted as MinLatency. // For empty itineraries, short-cirtuit the check and default to one cycle. @@ -629,7 +629,7 @@ static int computeDefOperandLatency( return 1; } else if(ItinData->isEmpty()) - return TII->defaultDefLatency(ItinData->SchedModel, DefMI); + return defaultDefLatency(ItinData->SchedModel, DefMI); // ...operand lookup required return -1; @@ -652,7 +652,7 @@ computeOperandLatency(const InstrItineraryData *ItinData, const MachineInstr *UseMI, unsigned UseIdx, bool FindMin) const { - int DefLatency = computeDefOperandLatency(this, ItinData, DefMI, FindMin); + int DefLatency = computeDefOperandLatency(ItinData, DefMI, FindMin); if (DefLatency >= 0) return DefLatency; diff --git a/lib/CodeGen/TargetSchedule.cpp b/lib/CodeGen/TargetSchedule.cpp index 42effb415b7..6611ef8f95d 100644 --- a/lib/CodeGen/TargetSchedule.cpp +++ b/lib/CodeGen/TargetSchedule.cpp @@ -14,6 +14,7 @@ #include "llvm/CodeGen/TargetSchedule.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" #include "llvm/Support/CommandLine.h" @@ -22,6 +23,9 @@ using namespace llvm; static cl::opt EnableSchedModel("schedmodel", cl::Hidden, cl::init(false), cl::desc("Use TargetSchedModel for latency lookup")); +static cl::opt EnableSchedItins("scheditins", cl::Hidden, cl::init(true), + cl::desc("Use InstrItineraryData for latency lookup")); + void TargetSchedModel::init(const MCSchedModel &sm, const TargetSubtargetInfo *sti, const TargetInstrInfo *tii) { @@ -30,3 +34,139 @@ void TargetSchedModel::init(const MCSchedModel &sm, TII = tii; STI->initInstrItins(InstrItins); } + +/// If we can determine the operand latency from the def only, without machine +/// model or itinerary lookup, do so. Otherwise return -1. +int TargetSchedModel::getDefLatency(const MachineInstr *DefMI, + bool FindMin) const { + + // Return a latency based on the itinerary properties and defining instruction + // if possible. Some common subtargets don't require per-operand latency, + // especially for minimum latencies. + if (FindMin) { + // If MinLatency is invalid, then use the itinerary for MinLatency. If no + // itinerary exists either, then use single cycle latency. + if (SchedModel.MinLatency < 0 + && !(EnableSchedItins && hasInstrItineraries())) { + return 1; + } + return SchedModel.MinLatency; + } + else if (!(EnableSchedModel && hasInstrSchedModel()) + && !(EnableSchedItins && hasInstrItineraries())) { + return TII->defaultDefLatency(&SchedModel, DefMI); + } + // ...operand lookup required + return -1; +} + +/// Return the MCSchedClassDesc for this instruction. Some SchedClasses require +/// evaluation of predicates that depend on instruction operands or flags. +const MCSchedClassDesc *TargetSchedModel:: +resolveSchedClass(const MachineInstr *MI) const { + + // Get the definition's scheduling class descriptor from this machine model. + unsigned SchedClass = MI->getDesc().getSchedClass(); + const MCSchedClassDesc *SCDesc = SchedModel.getSchedClassDesc(SchedClass); + +#ifndef NDEBUG + unsigned NIter = 0; +#endif + while (SCDesc->isVariant()) { + assert(++NIter < 6 && "Variants are nested deeper than the magic number"); + + SchedClass = STI->resolveSchedClass(SchedClass, MI, this); + SCDesc = SchedModel.getSchedClassDesc(SchedClass); + } + return SCDesc; +} + +/// Find the def index of this operand. This index maps to the machine model and +/// is independent of use operands. Def operands may be reordered with uses or +/// merged with uses without affecting the def index (e.g. before/after +/// regalloc). However, an instruction's def operands must never be reordered +/// with respect to each other. +static unsigned findDefIdx(const MachineInstr *MI, unsigned DefOperIdx) { + unsigned DefIdx = 0; + for (unsigned i = 0; i != DefOperIdx; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (MO.isReg() && MO.isDef()) + ++DefIdx; + } + return DefIdx; +} + +/// Find the use index of this operand. This is independent of the instruction's +/// def operands. +static unsigned findUseIdx(const MachineInstr *MI, unsigned UseOperIdx) { + unsigned UseIdx = 0; + for (unsigned i = 0; i != UseOperIdx; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (MO.isReg() && MO.isUse()) + ++UseIdx; + } + return UseIdx; +} + +// Top-level API for clients that know the operand indices. +unsigned TargetSchedModel::computeOperandLatency( + const MachineInstr *DefMI, unsigned DefOperIdx, + const MachineInstr *UseMI, unsigned UseOperIdx, + bool FindMin) const { + + int DefLatency = getDefLatency(DefMI, FindMin); + if (DefLatency >= 0) + return DefLatency; + + if (!FindMin && EnableSchedModel && hasInstrSchedModel()) { + const MCSchedClassDesc *SCDesc = resolveSchedClass(DefMI); + unsigned DefIdx = findDefIdx(DefMI, DefOperIdx); + if (DefIdx < SCDesc->NumWriteLatencyEntries) { + + // Lookup the definition's write latency in SubtargetInfo. + const MCWriteLatencyEntry *WLEntry = + STI->getWriteLatencyEntry(SCDesc, DefIdx); + unsigned WriteID = WLEntry->WriteResourceID; + unsigned Latency = WLEntry->Cycles; + if (!UseMI) + return Latency; + + // Lookup the use's latency adjustment in SubtargetInfo. + const MCSchedClassDesc *UseDesc = resolveSchedClass(UseMI); + if (UseDesc->NumReadAdvanceEntries == 0) + return Latency; + unsigned UseIdx = findUseIdx(UseMI, UseOperIdx); + return Latency - STI->getReadAdvanceCycles(UseDesc, UseIdx, WriteID); + } + // If DefIdx does not exist in the model (e.g. implicit defs), then return + // unit latency (defaultDefLatency may be too conservative). + // TODO: For unknown defs, we may want to use the subtarget's model + // for WAW latency here instead of 1 cycle. + assert((!SCDesc->isValid() || DefMI->getOperand(DefOperIdx).isImplicit()) && + "DefIdx exceeds machine model def operand list"); + return 1; + } + assert(EnableSchedItins && hasInstrItineraries() && + "operand latency requires itinerary"); + + int OperLatency = 0; + if (UseMI) { + OperLatency = + TII->getOperandLatency(&InstrItins, DefMI, DefOperIdx, UseMI, UseOperIdx); + } + else { + unsigned DefClass = DefMI->getDesc().getSchedClass(); + OperLatency = InstrItins.getOperandCycle(DefClass, DefOperIdx); + } + if (OperLatency >= 0) + return OperLatency; + + // No operand latency was found. + unsigned InstrLatency = TII->getInstrLatency(&InstrItins, DefMI); + + // Expected latency is the max of the stage latency and itinerary props. + if (!FindMin) + InstrLatency = std::max(InstrLatency, + TII->defaultDefLatency(&SchedModel, DefMI)); + return InstrLatency; +} diff --git a/lib/MC/MCSubtargetInfo.cpp b/lib/MC/MCSubtargetInfo.cpp index 34b7eeabbc8..ae2e8a1d6c9 100644 --- a/lib/MC/MCSubtargetInfo.cpp +++ b/lib/MC/MCSubtargetInfo.cpp @@ -113,5 +113,5 @@ MCSubtargetInfo::getInstrItineraryForCPU(StringRef CPU) const { /// Initialize an InstrItineraryData instance. void MCSubtargetInfo::initInstrItins(InstrItineraryData &InstrItins) const { InstrItins = - InstrItineraryData(0, Stages, OperandCycles, ForwardingPaths); + InstrItineraryData(CPUSchedModel, Stages, OperandCycles, ForwardingPaths); }