From e141a4960f702bef957b28abde3801ec64e32d87 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Mon, 8 Nov 2010 18:02:08 +0000 Subject: [PATCH] Adds support for spilling previously allocated live intervals to handle cases in which a register is unavailable for spill code. Adds LiveIntervalUnion::extract. While processing interferences on a live virtual register, reuses the same Query object for each physcial reg. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@118423 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/CodeGen/LiveIntervalUnion.cpp | 98 +++++++++++++--------- lib/CodeGen/LiveIntervalUnion.h | 49 ++++++++--- lib/CodeGen/RegAllocBase.h | 16 +++- lib/CodeGen/RegAllocBasic.cpp | 134 ++++++++++++++++++++++-------- 4 files changed, 208 insertions(+), 89 deletions(-) diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp index 9a47b3569b4..ec502cd51ec 100644 --- a/lib/CodeGen/LiveIntervalUnion.cpp +++ b/lib/CodeGen/LiveIntervalUnion.cpp @@ -20,6 +20,44 @@ #include using namespace llvm; +// Find the first segment in the range [segBegin,segments_.end()) that +// intersects with seg. If no intersection is found, return the first segI +// such that segI.start >= seg.end +// +// This logic is tied to the underlying LiveSegments data structure. For now, we +// use set::upper_bound to find the nearest starting position, +// then reverse iterate to find the first overlap. +// +// Upon entry we have segBegin.start < seg.end +// seg |--... +// \ . +// lvr ...-| +// +// After set::upper_bound, we have segI.start >= seg.start: +// seg |--... +// / +// lvr |--... +// +// Assuming intervals are disjoint, if an intersection exists, it must be the +// segment found or the one immediately preceeding it. We continue reverse +// iterating to return the first overlapping segment. +LiveIntervalUnion::SegmentIter +LiveIntervalUnion::upperBound(SegmentIter segBegin, + const LiveSegment &seg) { + assert(seg.end > segBegin->start && "segment iterator precondition"); + // get the next LIU segment such that segI->start is not less than seg.start + // + // FIXME: Once we have a B+tree, we can make good use of segBegin as a hint to + // upper_bound. For now, we're forced to search again from the root each time. + SegmentIter segI = segments_.upper_bound(seg); + while (segI != segBegin) { + --segI; + if (seg.start >= segI->end) + return ++segI; + } + return segI; +} + // Merge a LiveInterval's segments. Guarantee no overlaps. // // Consider coalescing adjacent segments to save space, even though it makes @@ -29,7 +67,7 @@ void LiveIntervalUnion::unify(LiveInterval &lvr) { SegmentIter segPos = segments_.begin(); for (LiveInterval::iterator lvrI = lvr.begin(), lvrEnd = lvr.end(); lvrI != lvrEnd; ++lvrI ) { - LiveSegment segment(lvrI->start, lvrI->end, lvr); + LiveSegment segment(lvrI->start, lvrI->end, &lvr); segPos = segments_.insert(segPos, segment); assert(*segPos == segment && "need equal val for equal key"); #ifndef NDEBUG @@ -47,40 +85,17 @@ void LiveIntervalUnion::unify(LiveInterval &lvr) { } } -// Low-level helper to find the first segment in the range [segI,segEnd) that -// intersects with a live virtual register segment, or segI.start >= lvr.end -// -// This logic is tied to the underlying LiveSegments data structure. For now, we -// use a binary search within the vector to find the nearest starting position, -// then reverse iterate to find the first overlap. -// -// Upon entry we have segI.start < lvrSeg.end -// seg |--... -// \ . -// lvr ...-| -// -// After binary search, we have segI.start >= lvrSeg.start: -// seg |--... -// / -// lvr |--... -// -// Assuming intervals are disjoint, if an intersection exists, it must be the -// segment found or immediately behind it. We continue reverse iterating to -// return the first overlap. -typedef LiveIntervalUnion::SegmentIter SegmentIter; -static SegmentIter upperBound(SegmentIter segBegin, - SegmentIter segEnd, - const LiveRange &lvrSeg) { - assert(lvrSeg.end > segBegin->start && "segment iterator precondition"); - // get the next LIU segment such that setg.start is not less than - // lvrSeg.start - SegmentIter segI = std::upper_bound(segBegin, segEnd, lvrSeg.start); - while (segI != segBegin) { - --segI; - if (lvrSeg.start >= segI->end) - return ++segI; +// Remove a live virtual register's segments from this union. +void LiveIntervalUnion::extract(const LiveInterval &lvr) { + // Remove each of the virtual register's live segments from the map. + SegmentIter segPos = segments_.begin(); + for (LiveInterval::const_iterator lvrI = lvr.begin(), lvrEnd = lvr.end(); + lvrI != lvrEnd; ++lvrI) { + LiveSegment seg(lvrI->start, lvrI->end, const_cast(&lvr)); + segPos = upperBound(segPos, seg); + assert(segPos != segments_.end() && "missing lvr segment"); + segments_.erase(segPos++); } - return segI; } // Private interface accessed by Query. @@ -102,8 +117,8 @@ static SegmentIter upperBound(SegmentIter segBegin, // Assumes that segments are sorted by start position in both // LiveInterval and LiveSegments. void LiveIntervalUnion::Query::findIntersection(InterferenceResult &ir) const { - LiveInterval::iterator lvrEnd = lvr_.end(); - SegmentIter liuEnd = liu_.end(); + LiveInterval::iterator lvrEnd = lvr_->end(); + SegmentIter liuEnd = liu_->end(); while (ir.liuSegI_ != liuEnd) { // Slowly advance the live virtual reg iterator until we surpass the next // segment in this union. If this is ever used for coalescing of fixed @@ -115,7 +130,8 @@ void LiveIntervalUnion::Query::findIntersection(InterferenceResult &ir) const { break; // lvrSegI_ may have advanced far beyond liuSegI_, // do a fast intersection test to "catch up" - ir.liuSegI_ = upperBound(ir.liuSegI_, liuEnd, *ir.lvrSegI_); + LiveSegment seg(ir.lvrSegI_->start, ir.lvrSegI_->end, lvr_); + ir.liuSegI_ = liu_->upperBound(ir.liuSegI_, seg); // Check if no liuSegI_ exists with lvrSegI_->start < liuSegI_.end if (ir.liuSegI_ == liuEnd) break; @@ -135,7 +151,7 @@ LiveIntervalUnion::Query::firstInterference() { if (firstInterference_ != LiveIntervalUnion::InterferenceResult()) { return firstInterference_; } - firstInterference_ = InterferenceResult(lvr_.begin(), liu_.begin()); + firstInterference_ = InterferenceResult(lvr_->begin(), liu_->begin()); findIntersection(firstInterference_); return firstInterference_; } @@ -147,12 +163,12 @@ bool LiveIntervalUnion::Query::nextInterference(InterferenceResult &ir) const { // Advance either the lvr or liu segment to ensure that we visit all unique // overlapping pairs. if (ir.lvrSegI_->end < ir.liuSegI_->end) { - if (++ir.lvrSegI_ == lvr_.end()) + if (++ir.lvrSegI_ == lvr_->end()) return false; } else { - if (++ir.liuSegI_ == liu_.end()) { - ir.lvrSegI_ = lvr_.end(); + if (++ir.liuSegI_ == liu_->end()) { + ir.lvrSegI_ = lvr_->end(); return false; } } diff --git a/lib/CodeGen/LiveIntervalUnion.h b/lib/CodeGen/LiveIntervalUnion.h index 0beadfa47ee..f0bce470875 100644 --- a/lib/CodeGen/LiveIntervalUnion.h +++ b/lib/CodeGen/LiveIntervalUnion.h @@ -38,8 +38,8 @@ struct LiveSegment { SlotIndex end; LiveInterval *liveVirtReg; - LiveSegment(SlotIndex s, SlotIndex e, LiveInterval &lvr) - : start(s), end(e), liveVirtReg(&lvr) {} + LiveSegment(SlotIndex s, SlotIndex e, LiveInterval *lvr) + : start(s), end(e), liveVirtReg(lvr) {} bool operator==(const LiveSegment &ls) const { return start == ls.start && end == ls.end && liveVirtReg == ls.liveVirtReg; @@ -111,12 +111,16 @@ public: SegmentIter begin() { return segments_.begin(); } SegmentIter end() { return segments_.end(); } + // Return an iterator to the first segment after or including begin that + // intersects with lvrSeg. + SegmentIter upperBound(SegmentIter begin, const LiveSegment &seg); + // Add a live virtual register to this union and merge its segments. // Holds a nonconst reference to the LVR for later maniplution. void unify(LiveInterval &lvr); - // FIXME: needed by RegAllocGreedy - //void extract(const LiveInterval &li); + // Remove a live virtual register's segments from this union. + void extract(const LiveInterval &lvr); /// Cache a single interference test result in the form of two intersecting /// segments. This allows efficiently iterating over the interferences. The @@ -143,7 +147,7 @@ public: const LiveInterval::iterator &lvrSegPos() const { return lvrSegI_; } // Access the liu segment. - const SegmentIter &liuSeg() const { return liuSegI_; } + const SegmentIter &liuSegPos() const { return liuSegI_; } bool operator==(const InterferenceResult &ir) const { return lvrSegI_ == ir.lvrSegI_ && liuSegI_ == ir.liuSegI_; @@ -156,18 +160,40 @@ public: /// Query interferences between a single live virtual register and a live /// interval union. class Query { - LiveIntervalUnion &liu_; - LiveInterval &lvr_; + LiveIntervalUnion *liu_; + LiveInterval *lvr_; InterferenceResult firstInterference_; // TBD: interfering vregs public: - Query(LiveInterval &lvr, LiveIntervalUnion &liu): liu_(liu), lvr_(lvr) {} + Query(): liu_(), lvr_() {} - LiveInterval &lvr() const { return lvr_; } + Query(LiveInterval *lvr, LiveIntervalUnion *liu): liu_(liu), lvr_(lvr) {} + + void clear() { + liu_ = NULL; + lvr_ = NULL; + firstInterference_ = InterferenceResult(); + } + + void init(LiveInterval *lvr, LiveIntervalUnion *liu) { + if (lvr_ == lvr) { + // We currently allow query objects to be reused acrossed live virtual + // registers, but always for the same live interval union. + assert(liu_ == liu && "inconsistent initialization"); + // Retain cached results, e.g. firstInterference. + return; + } + liu_ = liu; + lvr_ = lvr; + // Clear cached results. + firstInterference_ = InterferenceResult(); + } + + LiveInterval &lvr() const { assert(lvr_ && "uninitialized"); return *lvr_; } bool isInterference(const InterferenceResult &ir) const { - if (ir.lvrSegI_ != lvr_.end()) { + if (ir.lvrSegI_ != lvr_->end()) { assert(overlap(*ir.lvrSegI_, *ir.liuSegI_) && "invalid segment iterators"); return true; @@ -178,7 +204,8 @@ public: // Does this live virtual register interfere with the union. bool checkInterference() { return isInterference(firstInterference()); } - // First pair of interfering segments, or a noninterfering result. + // Get the first pair of interfering segments, or a noninterfering result. + // This initializes the firstInterference_ cache. InterferenceResult firstInterference(); // Treat the result as an iterator and advance to the next interfering pair diff --git a/lib/CodeGen/RegAllocBase.h b/lib/CodeGen/RegAllocBase.h index 1534c0d7eb8..f4ca9727384 100644 --- a/lib/CodeGen/RegAllocBase.h +++ b/lib/CodeGen/RegAllocBase.h @@ -94,6 +94,10 @@ protected: LiveIntervals *lis_; LIUArray physReg2liu_; + // Current queries, one per physreg. They must be reinitialized each time we + // query on a new live virtual register. + OwningArrayPtr queries_; + RegAllocBase(): tri_(0), vrm_(0), lis_(0) {} virtual ~RegAllocBase() {} @@ -120,9 +124,15 @@ protected: virtual void releaseMemory(); // Helper for checking interference between a live virtual register and a - // physical register, including all its register aliases. - bool checkPhysRegInterference(LiveIntervalUnion::Query &query, unsigned preg); - + // physical register, including all its register aliases. If an interference + // exists, return the interfering register, which may be preg or an alias. + unsigned checkPhysRegInterference(LiveInterval& lvr, unsigned preg); + + // Helper that spills all live virtual registers currently unified under preg + // that interfere with the most recently queried lvr. + void spillInterferences(unsigned preg, + SmallVectorImpl &splitLVRs); + private: void seedLiveVirtRegs(LiveVirtRegQueue &lvrQ); }; diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index 6c592c8e25e..a2f9ea274b9 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -17,10 +17,12 @@ #include "RegAllocBase.h" #include "RenderMachineFunction.h" #include "Spiller.h" +#include "VirtRegMap.h" #include "VirtRegRewriter.h" #include "llvm/Function.h" #include "llvm/PassAnalysisSupport.h" #include "llvm/CodeGen/CalcSpillWeights.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" @@ -31,13 +33,10 @@ #include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" - -#include "VirtRegMap.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/Target/TargetRegisterInfo.h" - +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -84,6 +83,9 @@ public: virtual unsigned selectOrSplit(LiveInterval &lvr, SmallVectorImpl &splitLVRs); + void spillInterferences(unsigned preg, + SmallVectorImpl &splitLVRs); + /// Perform register allocation. virtual bool runOnMachineFunction(MachineFunction &mf); @@ -170,6 +172,8 @@ void RegAllocBase::init(const TargetRegisterInfo &tri, VirtRegMap &vrm, vrm_ = &vrm; lis_ = &lis; physReg2liu_.init(tri_->getNumRegs()); + // Cache an interferece query for each physical reg + queries_.reset(new LiveIntervalUnion::Query[physReg2liu_.numRegs()]); } void RegAllocBase::LIUArray::clear() { @@ -238,38 +242,61 @@ void RegAllocBase::allocatePhysRegs() { LVRVec splitLVRs; unsigned availablePhysReg = selectOrSplit(*lvr, splitLVRs); if (availablePhysReg) { - assert(splitLVRs.empty() && "inconsistent splitting"); + DEBUG(dbgs() << "allocating: " << tri_->getName(availablePhysReg) << + " " << lvr << '\n'); assert(!vrm_->hasPhys(lvr->reg) && "duplicate vreg in interval unions"); vrm_->assignVirt2Phys(lvr->reg, availablePhysReg); physReg2liu_[availablePhysReg].unify(*lvr); } - else { - for (LVRVec::iterator lvrI = splitLVRs.begin(), lvrEnd = splitLVRs.end(); - lvrI != lvrEnd; ++lvrI) { - assert(TargetRegisterInfo::isVirtualRegister((*lvrI)->reg) && - "expect split value in virtual register"); - lvrQ.push(*lvrI); - } + for (LVRVec::iterator lvrI = splitLVRs.begin(), lvrEnd = splitLVRs.end(); + lvrI != lvrEnd; ++lvrI) { + DEBUG(dbgs() << "queuing new interval: " << **lvrI << "\n"); + assert(TargetRegisterInfo::isVirtualRegister((*lvrI)->reg) && + "expect split value in virtual register"); + lvrQ.push(*lvrI); } } } // Check if this live virtual reg interferes with a physical register. If not, // then check for interference on each register that aliases with the physical -// register. -bool RegAllocBase::checkPhysRegInterference(LiveIntervalUnion::Query &query, - unsigned preg) { - if (query.checkInterference()) - return true; +// register. Return the interfering register. +unsigned RegAllocBase::checkPhysRegInterference(LiveInterval &lvr, + unsigned preg) { + queries_[preg].init(&lvr, &physReg2liu_[preg]); + if (queries_[preg].checkInterference()) + return preg; for (const unsigned *asI = tri_->getAliasSet(preg); *asI; ++asI) { - // We assume it's very unlikely for a register in the alias set to also be - // in the original register class. So we don't bother caching the - // interference. - LiveIntervalUnion::Query subQuery(query.lvr(), physReg2liu_[*asI] ); - if (subQuery.checkInterference()) - return true; + queries_[*asI].init(&lvr, &physReg2liu_[*asI]); + if (queries_[*asI].checkInterference()) + return *asI; } - return false; + return 0; +} + +// Spill all live virtual registers currently unified under preg that interfere +// with lvr. +void RABasic::spillInterferences(unsigned preg, + SmallVectorImpl &splitLVRs) { + SmallPtrSet spilledLVRs; + LiveIntervalUnion::Query &query = queries_[preg]; + LiveIntervalUnion::InterferenceResult ir = query.firstInterference(); + assert(query.isInterference(ir) && "expect interference"); + do { + LiveInterval *lvr = ir.liuSegPos()->liveVirtReg; + if (!spilledLVRs.insert(lvr)) continue; + // Spill the previously allocated lvr. + SmallVector spillIs; // ignored + spiller_->spill(lvr, splitLVRs, spillIs); + } while (query.nextInterference(ir)); + for (SmallPtrSetIterator lvrI = spilledLVRs.begin(), + lvrEnd = spilledLVRs.end(); + lvrI != lvrEnd; ++lvrI ) { + // Deallocate the interfering lvr by removing it from the preg union. + physReg2liu_[preg].extract(**lvrI); + } + // After extracting segments, the query's results are invalid. + query.clear(); } //===----------------------------------------------------------------------===// @@ -289,24 +316,59 @@ bool RegAllocBase::checkPhysRegInterference(LiveIntervalUnion::Query &query, // minimal, there is no value in caching them. unsigned RABasic::selectOrSplit(LiveInterval &lvr, SmallVectorImpl &splitLVRs) { + // Accumulate the min spill cost among the interferences, in case we spill. + unsigned minSpillReg = 0; + unsigned minSpillAlias = 0; + float minSpillWeight = lvr.weight; + // Check for an available reg in this class. const TargetRegisterClass *trc = mri_->getRegClass(lvr.reg); for (TargetRegisterClass::iterator trcI = trc->allocation_order_begin(*mf_), trcEnd = trc->allocation_order_end(*mf_); trcI != trcEnd; ++trcI) { unsigned preg = *trcI; - LiveIntervalUnion::Query query(lvr, physReg2liu_[preg]); - if (!checkPhysRegInterference(query, preg)) { - DEBUG(dbgs() << "\tallocating: " << tri_->getName(preg) << lvr << '\n'); + unsigned interfReg = checkPhysRegInterference(lvr, preg); + if (interfReg == 0) { return preg; } + LiveIntervalUnion::InterferenceResult interf = + queries_[interfReg].firstInterference(); + float interfWeight = interf.liuSegPos()->liveVirtReg->weight; + if (interfWeight < minSpillWeight ) { + minSpillReg = interfReg; + minSpillAlias = preg; + minSpillWeight = interfWeight; + } } - DEBUG(dbgs() << "\tspilling: " << lvr << '\n'); - SmallVector spillIs; // ignored - spiller_->spill(&lvr, splitLVRs, spillIs); + if (minSpillReg == 0) { + DEBUG(dbgs() << "spilling: " << lvr << '\n'); + SmallVector spillIs; // ignored + spiller_->spill(&lvr, splitLVRs, spillIs); + // The live virtual register requesting to be allocated was spilled. So tell + // the caller not to allocate anything for this round. + return 0; + } + // Free the cheapest physical register. + spillInterferences(minSpillReg, splitLVRs); + // Tell the caller to allocate to this newly freed physical register. + assert(minSpillAlias != 0 && "need a free register after spilling"); + // We just spilled the first register that interferes with minSpillAlias. We + // now assume minSpillAlias is free because only one register alias may + // interfere at a time. e.g. we ignore predication. + unsigned interfReg = checkPhysRegInterference(lvr, minSpillAlias); + if (interfReg != 0) { + dbgs() << "spilling cannot free " << tri_->getName(minSpillAlias) << + " for " << lvr.reg << " with interference " << + *queries_[interfReg].firstInterference().liuSegPos()->liveVirtReg << "\n"; + llvm_unreachable("Interference after spill."); + } + return minSpillAlias; +} - // FIXME: update LiveStacks - return 0; +namespace llvm { +Spiller *createInlineSpiller(MachineFunctionPass &pass, + MachineFunction &mf, + VirtRegMap &vrm); } bool RABasic::runOnMachineFunction(MachineFunction &mf) { @@ -323,6 +385,10 @@ bool RABasic::runOnMachineFunction(MachineFunction &mf) { RegAllocBase::init(*tm_->getRegisterInfo(), getAnalysis(), getAnalysis()); + // We may want to force InlineSpiller for this register allocator. For + // now we're also experimenting with the standard spiller. + // + //spiller_.reset(createInlineSpiller(*this, *mf_, *vrm_)); spiller_.reset(createSpiller(*this, *mf_, *vrm_)); allocatePhysRegs();