|
|
|
@ -35,7 +35,7 @@
|
|
|
|
|
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
|
|
#define DEBUG_TYPE "execution-fix"
|
|
|
|
|
#define DEBUG_TYPE "execution-deps-fix"
|
|
|
|
|
|
|
|
|
|
/// A DomainValue is a bit like LiveIntervals' ValNo, but it also keeps track
|
|
|
|
|
/// of execution domains.
|
|
|
|
@ -129,7 +129,7 @@ struct LiveReg {
|
|
|
|
|
} // anonymous namespace
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
class ExeDepsFix : public MachineFunctionPass {
|
|
|
|
|
class ExecutionDepsFix : public MachineFunctionPass {
|
|
|
|
|
static char ID;
|
|
|
|
|
SpecificBumpPtrAllocator<DomainValue> Allocator;
|
|
|
|
|
SmallVector<DomainValue*,16> Avail;
|
|
|
|
@ -177,7 +177,7 @@ class ExeDepsFix : public MachineFunctionPass {
|
|
|
|
|
/// The first instruction in each basic block is 0.
|
|
|
|
|
int CurInstr;
|
|
|
|
|
public:
|
|
|
|
|
ExeDepsFix(const TargetRegisterClass *rc)
|
|
|
|
|
ExecutionDepsFix(const TargetRegisterClass *rc)
|
|
|
|
|
: MachineFunctionPass(ID), RC(rc), NumRegs(RC->getNumRegs()) {}
|
|
|
|
|
|
|
|
|
|
void getAnalysisUsage(AnalysisUsage &AU) const override {
|
|
|
|
@ -229,18 +229,18 @@ private:
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
char ExeDepsFix::ID = 0;
|
|
|
|
|
char ExecutionDepsFix::ID = 0;
|
|
|
|
|
|
|
|
|
|
/// Translate TRI register number to a list of indices into our smaller tables
|
|
|
|
|
/// of interesting registers.
|
|
|
|
|
iterator_range<SmallVectorImpl<int>::const_iterator>
|
|
|
|
|
ExeDepsFix::regIndices(unsigned Reg) const {
|
|
|
|
|
ExecutionDepsFix::regIndices(unsigned Reg) const {
|
|
|
|
|
assert(Reg < AliasMap.size() && "Invalid register");
|
|
|
|
|
const auto &Entry = AliasMap[Reg];
|
|
|
|
|
return make_range(Entry.begin(), Entry.end());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
DomainValue *ExeDepsFix::alloc(int domain) {
|
|
|
|
|
DomainValue *ExecutionDepsFix::alloc(int domain) {
|
|
|
|
|
DomainValue *dv = Avail.empty() ?
|
|
|
|
|
new(Allocator.Allocate()) DomainValue :
|
|
|
|
|
Avail.pop_back_val();
|
|
|
|
@ -253,7 +253,7 @@ DomainValue *ExeDepsFix::alloc(int domain) {
|
|
|
|
|
|
|
|
|
|
/// Release a reference to DV. When the last reference is released,
|
|
|
|
|
/// collapse if needed.
|
|
|
|
|
void ExeDepsFix::release(DomainValue *DV) {
|
|
|
|
|
void ExecutionDepsFix::release(DomainValue *DV) {
|
|
|
|
|
while (DV) {
|
|
|
|
|
assert(DV->Refs && "Bad DomainValue");
|
|
|
|
|
if (--DV->Refs)
|
|
|
|
@ -273,7 +273,7 @@ void ExeDepsFix::release(DomainValue *DV) {
|
|
|
|
|
|
|
|
|
|
/// Follow the chain of dead DomainValues until a live DomainValue is reached.
|
|
|
|
|
/// Update the referenced pointer when necessary.
|
|
|
|
|
DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) {
|
|
|
|
|
DomainValue *ExecutionDepsFix::resolve(DomainValue *&DVRef) {
|
|
|
|
|
DomainValue *DV = DVRef;
|
|
|
|
|
if (!DV || !DV->Next)
|
|
|
|
|
return DV;
|
|
|
|
@ -290,7 +290,7 @@ DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Set LiveRegs[rx] = dv, updating reference counts.
|
|
|
|
|
void ExeDepsFix::setLiveReg(int rx, DomainValue *dv) {
|
|
|
|
|
void ExecutionDepsFix::setLiveReg(int rx, DomainValue *dv) {
|
|
|
|
|
assert(unsigned(rx) < NumRegs && "Invalid index");
|
|
|
|
|
assert(LiveRegs && "Must enter basic block first.");
|
|
|
|
|
|
|
|
|
@ -302,7 +302,7 @@ void ExeDepsFix::setLiveReg(int rx, DomainValue *dv) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Kill register rx, recycle or collapse any DomainValue.
|
|
|
|
|
void ExeDepsFix::kill(int rx) {
|
|
|
|
|
void ExecutionDepsFix::kill(int rx) {
|
|
|
|
|
assert(unsigned(rx) < NumRegs && "Invalid index");
|
|
|
|
|
assert(LiveRegs && "Must enter basic block first.");
|
|
|
|
|
if (!LiveRegs[rx].Value)
|
|
|
|
@ -313,7 +313,7 @@ void ExeDepsFix::kill(int rx) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Force register rx into domain.
|
|
|
|
|
void ExeDepsFix::force(int rx, unsigned domain) {
|
|
|
|
|
void ExecutionDepsFix::force(int rx, unsigned domain) {
|
|
|
|
|
assert(unsigned(rx) < NumRegs && "Invalid index");
|
|
|
|
|
assert(LiveRegs && "Must enter basic block first.");
|
|
|
|
|
if (DomainValue *dv = LiveRegs[rx].Value) {
|
|
|
|
@ -336,7 +336,7 @@ void ExeDepsFix::force(int rx, unsigned domain) {
|
|
|
|
|
|
|
|
|
|
/// Collapse open DomainValue into given domain. If there are multiple
|
|
|
|
|
/// registers using dv, they each get a unique collapsed DomainValue.
|
|
|
|
|
void ExeDepsFix::collapse(DomainValue *dv, unsigned domain) {
|
|
|
|
|
void ExecutionDepsFix::collapse(DomainValue *dv, unsigned domain) {
|
|
|
|
|
assert(dv->hasDomain(domain) && "Cannot collapse");
|
|
|
|
|
|
|
|
|
|
// Collapse all the instructions.
|
|
|
|
@ -352,7 +352,7 @@ void ExeDepsFix::collapse(DomainValue *dv, unsigned domain) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// All instructions and registers in B are moved to A, and B is released.
|
|
|
|
|
bool ExeDepsFix::merge(DomainValue *A, DomainValue *B) {
|
|
|
|
|
bool ExecutionDepsFix::merge(DomainValue *A, DomainValue *B) {
|
|
|
|
|
assert(!A->isCollapsed() && "Cannot merge into collapsed");
|
|
|
|
|
assert(!B->isCollapsed() && "Cannot merge from collapsed");
|
|
|
|
|
if (A == B)
|
|
|
|
@ -378,7 +378,7 @@ bool ExeDepsFix::merge(DomainValue *A, DomainValue *B) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Set up LiveRegs by merging predecessor live-out values.
|
|
|
|
|
void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
void ExecutionDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
// Reset instruction counter in each basic block.
|
|
|
|
|
CurInstr = 0;
|
|
|
|
|
|
|
|
|
@ -456,7 +456,7 @@ void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
<< (!isBlockDone(MBB) ? ": incomplete\n" : ": all preds known\n"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ExeDepsFix::leaveBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
void ExecutionDepsFix::leaveBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
assert(LiveRegs && "Must enter basic block first.");
|
|
|
|
|
LiveReg *OldOutRegs = MBBInfos[MBB].OutRegs;
|
|
|
|
|
// Save register clearances at end of MBB - used by enterBasicBlock().
|
|
|
|
@ -478,7 +478,7 @@ void ExeDepsFix::leaveBasicBlock(MachineBasicBlock *MBB) {
|
|
|
|
|
LiveRegs = nullptr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool ExeDepsFix::visitInstr(MachineInstr *MI) {
|
|
|
|
|
bool ExecutionDepsFix::visitInstr(MachineInstr *MI) {
|
|
|
|
|
// Update instructions with explicit execution domains.
|
|
|
|
|
std::pair<uint16_t, uint16_t> DomP = TII->getExecutionDomain(*MI);
|
|
|
|
|
if (DomP.first) {
|
|
|
|
@ -494,8 +494,8 @@ bool ExeDepsFix::visitInstr(MachineInstr *MI) {
|
|
|
|
|
/// \brief Helps avoid false dependencies on undef registers by updating the
|
|
|
|
|
/// machine instructions' undef operand to use a register that the instruction
|
|
|
|
|
/// is truly dependent on, or use a register with clearance higher than Pref.
|
|
|
|
|
void ExeDepsFix::pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx,
|
|
|
|
|
unsigned Pref) {
|
|
|
|
|
void ExecutionDepsFix::pickBestRegisterForUndef(MachineInstr *MI,
|
|
|
|
|
unsigned OpIdx, unsigned Pref) {
|
|
|
|
|
MachineOperand &MO = MI->getOperand(OpIdx);
|
|
|
|
|
assert(MO.isUndef() && "Expected undef machine operand");
|
|
|
|
|
|
|
|
|
@ -547,8 +547,8 @@ void ExeDepsFix::pickBestRegisterForUndef(MachineInstr *MI, unsigned OpIdx,
|
|
|
|
|
|
|
|
|
|
/// \brief Return true to if it makes sense to break dependence on a partial def
|
|
|
|
|
/// or undef use.
|
|
|
|
|
bool ExeDepsFix::shouldBreakDependence(MachineInstr *MI, unsigned OpIdx,
|
|
|
|
|
unsigned Pref) {
|
|
|
|
|
bool ExecutionDepsFix::shouldBreakDependence(MachineInstr *MI, unsigned OpIdx,
|
|
|
|
|
unsigned Pref) {
|
|
|
|
|
unsigned reg = MI->getOperand(OpIdx).getReg();
|
|
|
|
|
for (int rx : regIndices(reg)) {
|
|
|
|
|
unsigned Clearance = CurInstr - LiveRegs[rx].Def;
|
|
|
|
@ -568,8 +568,8 @@ bool ExeDepsFix::shouldBreakDependence(MachineInstr *MI, unsigned OpIdx,
|
|
|
|
|
// If Kill is set, also kill off DomainValues clobbered by the defs.
|
|
|
|
|
//
|
|
|
|
|
// Also break dependencies on partial defs and undef uses.
|
|
|
|
|
void ExeDepsFix::processDefs(MachineInstr *MI, bool breakDependency,
|
|
|
|
|
bool Kill) {
|
|
|
|
|
void ExecutionDepsFix::processDefs(MachineInstr *MI, bool breakDependency,
|
|
|
|
|
bool Kill) {
|
|
|
|
|
assert(!MI->isDebugValue() && "Won't process debug values");
|
|
|
|
|
|
|
|
|
|
// Break dependence on undef uses. Do this before updating LiveRegs below.
|
|
|
|
@ -621,7 +621,7 @@ void ExeDepsFix::processDefs(MachineInstr *MI, bool breakDependency,
|
|
|
|
|
/// only do it on demand. Note that the occurrence of undefined register reads
|
|
|
|
|
/// that should be broken is very rare, but when they occur we may have many in
|
|
|
|
|
/// a single block.
|
|
|
|
|
void ExeDepsFix::processUndefReads(MachineBasicBlock *MBB) {
|
|
|
|
|
void ExecutionDepsFix::processUndefReads(MachineBasicBlock *MBB) {
|
|
|
|
|
if (UndefReads.empty())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
@ -654,7 +654,7 @@ void ExeDepsFix::processUndefReads(MachineBasicBlock *MBB) {
|
|
|
|
|
|
|
|
|
|
// A hard instruction only works in one domain. All input registers will be
|
|
|
|
|
// forced into that domain.
|
|
|
|
|
void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) {
|
|
|
|
|
void ExecutionDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) {
|
|
|
|
|
// Collapse all uses.
|
|
|
|
|
for (unsigned i = mi->getDesc().getNumDefs(),
|
|
|
|
|
e = mi->getDesc().getNumOperands(); i != e; ++i) {
|
|
|
|
@ -677,7 +677,7 @@ void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A soft instruction can be changed to work in other domains given by mask.
|
|
|
|
|
void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
|
|
|
|
|
void ExecutionDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
|
|
|
|
|
// Bitmask of available domains for this instruction after taking collapsed
|
|
|
|
|
// operands into account.
|
|
|
|
|
unsigned available = mask;
|
|
|
|
@ -788,7 +788,8 @@ void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ExeDepsFix::processBasicBlock(MachineBasicBlock *MBB, bool PrimaryPass) {
|
|
|
|
|
void ExecutionDepsFix::processBasicBlock(MachineBasicBlock *MBB,
|
|
|
|
|
bool PrimaryPass) {
|
|
|
|
|
enterBasicBlock(MBB);
|
|
|
|
|
// If this block is not done, it makes little sense to make any decisions
|
|
|
|
|
// based on clearance information. We need to make a second pass anyway,
|
|
|
|
@ -808,13 +809,13 @@ void ExeDepsFix::processBasicBlock(MachineBasicBlock *MBB, bool PrimaryPass) {
|
|
|
|
|
leaveBasicBlock(MBB);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool ExeDepsFix::isBlockDone(MachineBasicBlock *MBB) {
|
|
|
|
|
bool ExecutionDepsFix::isBlockDone(MachineBasicBlock *MBB) {
|
|
|
|
|
return MBBInfos[MBB].PrimaryCompleted &&
|
|
|
|
|
MBBInfos[MBB].IncomingCompleted == MBBInfos[MBB].PrimaryIncoming &&
|
|
|
|
|
MBBInfos[MBB].IncomingProcessed == MBB->pred_size();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void ExeDepsFix::updateSuccessors(MachineBasicBlock *MBB, bool Primary) {
|
|
|
|
|
void ExecutionDepsFix::updateSuccessors(MachineBasicBlock *MBB, bool Primary) {
|
|
|
|
|
bool Done = isBlockDone(MBB);
|
|
|
|
|
for (auto *Succ : MBB->successors()) {
|
|
|
|
|
if (!isBlockDone(Succ)) {
|
|
|
|
@ -834,7 +835,7 @@ void ExeDepsFix::updateSuccessors(MachineBasicBlock *MBB, bool Primary) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
|
|
|
|
|
bool ExecutionDepsFix::runOnMachineFunction(MachineFunction &mf) {
|
|
|
|
|
if (skipFunction(*mf.getFunction()))
|
|
|
|
|
return false;
|
|
|
|
|
MF = &mf;
|
|
|
|
@ -951,7 +952,6 @@ bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
FunctionPass *
|
|
|
|
|
llvm::createExecutionDependencyFixPass(const TargetRegisterClass *RC) {
|
|
|
|
|
return new ExeDepsFix(RC);
|
|
|
|
|
FunctionPass *llvm::createExecutionDepsFixPass(const TargetRegisterClass *RC) {
|
|
|
|
|
return new ExecutionDepsFix(RC);
|
|
|
|
|
}
|
|
|
|
|