Extend the inline cost calculation to account for bonuses due to

correlated pairs of pointer arguments at the callsite. This is designed
to recognize the common C++ idiom of begin/end pointer pairs when the
end pointer is a constant offset from the begin pointer. With the
C-based idiom of a pointer and size, the inline cost saw the constant
size calculation, and this provides the same level of information for
begin/end pairs.

In order to propagate this information we have to search for candidate
operations on a pair of pointer function arguments (or derived from
them) which would be simplified if the pointers had a known constant
offset. Then the callsite analysis looks for such pointer pairs in the
argument list, and applies the appropriate bonus.

This helps LLVM detect that half of bounds-checked STL algorithms
(such as hash_combine_range, and some hybrid sort implementations)
disappear when inlined with a constant size input. However, it's not
a complete fix due the inaccuracy of our cost metric for constants in
general. I'm looking into that next.

Benchmarks showed no significant code size change, and very minor
performance changes. However, specific code such as hashing is showing
significantly cleaner inlining decisions.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@152752 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Chandler Carruth 2012-03-14 23:19:53 +00:00
parent 8b11fdd8bb
commit 274d377ea6
5 changed files with 168 additions and 16 deletions

View File

@ -110,6 +110,10 @@ namespace llvm {
/// entry here. /// entry here.
std::vector<ArgInfo> ArgumentWeights; std::vector<ArgInfo> ArgumentWeights;
/// PointerArgPairWeights - Weights to use when giving an inline bonus to
/// a call site due to correlated pairs of pointers.
DenseMap<std::pair<unsigned, unsigned>, unsigned> PointerArgPairWeights;
/// countCodeReductionForConstant - Figure out an approximation for how /// countCodeReductionForConstant - Figure out an approximation for how
/// many instructions will be constant folded if the specified value is /// many instructions will be constant folded if the specified value is
/// constant. /// constant.
@ -122,6 +126,18 @@ namespace llvm {
unsigned countCodeReductionForAlloca(const CodeMetrics &Metrics, unsigned countCodeReductionForAlloca(const CodeMetrics &Metrics,
Value *V); Value *V);
/// countCodeReductionForPointerPair - Count the bonus to apply to an
/// inline call site where a pair of arguments are pointers and one
/// argument is a constant offset from the other. The idea is to
/// recognize a common C++ idiom where a begin and end iterator are
/// actually pointers, and many operations on the pair of them will be
/// constants if the function is called with arguments that have
/// a constant offset.
void countCodeReductionForPointerPair(
const CodeMetrics &Metrics,
DenseMap<Value *, unsigned> &PointerArgs,
Value *V, unsigned ArgIdx);
/// analyzeFunction - Add information about the specified function /// analyzeFunction - Add information about the specified function
/// to the current structure. /// to the current structure.
void analyzeFunction(Function *F, const TargetData *TD); void analyzeFunction(Function *F, const TargetData *TD);

View File

@ -271,13 +271,13 @@ public:
return const_cast<Value*>(this)->stripPointerCasts(); return const_cast<Value*>(this)->stripPointerCasts();
} }
/// stripConstantOffsets - This method strips off unneeded pointer casts and /// stripInBoundsConstantOffsets - This method strips off unneeded pointer casts and
/// all-constant GEPs from the specified value, returning the original /// all-constant GEPs from the specified value, returning the original
/// pointer value. If this is called on a non-pointer value, it returns /// pointer value. If this is called on a non-pointer value, it returns
/// 'this'. /// 'this'.
Value *stripConstantOffsets(); Value *stripInBoundsConstantOffsets();
const Value *stripConstantOffsets() const { const Value *stripInBoundsConstantOffsets() const {
return const_cast<Value*>(this)->stripConstantOffsets(); return const_cast<Value*>(this)->stripInBoundsConstantOffsets();
} }
/// stripInBoundsOffsets - This method strips off unneeded pointer casts and /// stripInBoundsOffsets - This method strips off unneeded pointer casts and

View File

@ -381,6 +381,67 @@ unsigned InlineCostAnalyzer::FunctionInfo::countCodeReductionForAlloca(
return Reduction + (CanSROAAlloca ? SROAReduction : 0); return Reduction + (CanSROAAlloca ? SROAReduction : 0);
} }
void InlineCostAnalyzer::FunctionInfo::countCodeReductionForPointerPair(
const CodeMetrics &Metrics, DenseMap<Value *, unsigned> &PointerArgs,
Value *V, unsigned ArgIdx) {
SmallVector<Value *, 4> Worklist;
Worklist.push_back(V);
do {
Value *V = Worklist.pop_back_val();
for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
UI != E; ++UI){
Instruction *I = cast<Instruction>(*UI);
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
// If the GEP has variable indices, we won't be able to do much with it.
if (!GEP->hasAllConstantIndices())
continue;
// Unless the GEP is in-bounds, some comparisons will be non-constant.
// Fortunately, the real-world cases where this occurs uses in-bounds
// GEPs, and so we restrict the optimization to them here.
if (!GEP->isInBounds())
continue;
// Constant indices just change the constant offset. Add the resulting
// value both to our worklist for this argument, and to the set of
// viable paired values with future arguments.
PointerArgs[GEP] = ArgIdx;
Worklist.push_back(GEP);
continue;
}
// Track pointer through casts. Even when the result is not a pointer, it
// remains a constant relative to constants derived from other constant
// pointers.
if (CastInst *CI = dyn_cast<CastInst>(I)) {
PointerArgs[CI] = ArgIdx;
Worklist.push_back(CI);
continue;
}
// There are two instructions which produce a strict constant value when
// applied to two related pointer values. Ignore everything else.
if (!isa<ICmpInst>(I) && I->getOpcode() != Instruction::Sub)
continue;
assert(I->getNumOperands() == 2);
// Ensure that the two operands are in our set of potentially paired
// pointers (or are derived from them).
Value *OtherArg = I->getOperand(0);
if (OtherArg == V)
OtherArg = I->getOperand(1);
DenseMap<Value *, unsigned>::const_iterator ArgIt
= PointerArgs.find(OtherArg);
if (ArgIt == PointerArgs.end())
continue;
assert(ArgIt->second < ArgIdx);
PointerArgPairWeights[std::make_pair(ArgIt->second, ArgIdx)]
+= countCodeReductionForConstant(Metrics, I);
}
} while (!Worklist.empty());
}
/// analyzeFunction - Fill in the current structure with information gleaned /// analyzeFunction - Fill in the current structure with information gleaned
/// from the specified function. /// from the specified function.
void CodeMetrics::analyzeFunction(Function *F, const TargetData *TD) { void CodeMetrics::analyzeFunction(Function *F, const TargetData *TD) {
@ -409,12 +470,25 @@ void InlineCostAnalyzer::FunctionInfo::analyzeFunction(Function *F,
if (Metrics.NumRets==1) if (Metrics.NumRets==1)
--Metrics.NumInsts; --Metrics.NumInsts;
// Check out all of the arguments to the function, figuring out how much
// code can be eliminated if one of the arguments is a constant.
ArgumentWeights.reserve(F->arg_size()); ArgumentWeights.reserve(F->arg_size());
for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) DenseMap<Value *, unsigned> PointerArgs;
unsigned ArgIdx = 0;
for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;
++I, ++ArgIdx) {
// Count how much code can be eliminated if one of the arguments is
// a constant or an alloca.
ArgumentWeights.push_back(ArgInfo(countCodeReductionForConstant(Metrics, I), ArgumentWeights.push_back(ArgInfo(countCodeReductionForConstant(Metrics, I),
countCodeReductionForAlloca(Metrics, I))); countCodeReductionForAlloca(Metrics, I)));
// If the argument is a pointer, also check for pairs of pointers where
// knowing a fixed offset between them allows simplification. This pattern
// arises mostly due to STL algorithm patterns where pointers are used as
// random access iterators.
if (!I->getType()->isPointerTy())
continue;
PointerArgs[I] = ArgIdx;
countCodeReductionForPointerPair(Metrics, PointerArgs, I, ArgIdx);
}
} }
/// NeverInline - returns true if the function should never be inlined into /// NeverInline - returns true if the function should never be inlined into
@ -563,6 +637,15 @@ int InlineCostAnalyzer::getInlineSize(CallSite CS, Function *Callee) {
InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight; InlineCost -= CalleeFI->ArgumentWeights[ArgNo].ConstantWeight;
} }
const DenseMap<std::pair<unsigned, unsigned>, unsigned> &ArgPairWeights
= CalleeFI->PointerArgPairWeights;
for (DenseMap<std::pair<unsigned, unsigned>, unsigned>::const_iterator I
= ArgPairWeights.begin(), E = ArgPairWeights.end();
I != E; ++I)
if (CS.getArgument(I->first.first)->stripInBoundsConstantOffsets() ==
CS.getArgument(I->first.second)->stripInBoundsConstantOffsets())
InlineCost -= I->second;
// Each argument passed in has a cost at both the caller and the callee // Each argument passed in has a cost at both the caller and the callee
// sides. Measurements show that each argument costs about the same as an // sides. Measurements show that each argument costs about the same as an
// instruction. // instruction.

View File

@ -321,9 +321,8 @@ namespace {
// Various metrics for how much to strip off of pointers. // Various metrics for how much to strip off of pointers.
enum PointerStripKind { enum PointerStripKind {
PSK_ZeroIndices, PSK_ZeroIndices,
PSK_ConstantIndices, PSK_InBoundsConstantIndices,
PSK_InBounds, PSK_InBounds
PSK_All
}; };
template <PointerStripKind StripKind> template <PointerStripKind StripKind>
@ -343,16 +342,14 @@ static Value *stripPointerCastsAndOffsets(Value *V) {
if (!GEP->hasAllZeroIndices()) if (!GEP->hasAllZeroIndices())
return V; return V;
break; break;
case PSK_ConstantIndices: case PSK_InBoundsConstantIndices:
if (!GEP->hasAllConstantIndices()) if (!GEP->hasAllConstantIndices())
return V; return V;
break; // fallthrough
case PSK_InBounds: case PSK_InBounds:
if (!GEP->isInBounds()) if (!GEP->isInBounds())
return V; return V;
break; break;
case PSK_All:
break;
} }
V = GEP->getPointerOperand(); V = GEP->getPointerOperand();
} else if (Operator::getOpcode(V) == Instruction::BitCast) { } else if (Operator::getOpcode(V) == Instruction::BitCast) {
@ -375,8 +372,8 @@ Value *Value::stripPointerCasts() {
return stripPointerCastsAndOffsets<PSK_ZeroIndices>(this); return stripPointerCastsAndOffsets<PSK_ZeroIndices>(this);
} }
Value *Value::stripConstantOffsets() { Value *Value::stripInBoundsConstantOffsets() {
return stripPointerCastsAndOffsets<PSK_ConstantIndices>(this); return stripPointerCastsAndOffsets<PSK_InBoundsConstantIndices>(this);
} }
Value *Value::stripInBoundsOffsets() { Value *Value::stripInBoundsOffsets() {

View File

@ -0,0 +1,56 @@
; RUN: opt -inline < %s -S -o - -inline-threshold=10 | FileCheck %s
define i32 @outer1() {
; CHECK: @outer1
; CHECK-NOT: call
; CHECK: ret i32
%ptr = alloca i32
%ptr1 = getelementptr inbounds i32* %ptr, i32 0
%ptr2 = getelementptr inbounds i32* %ptr, i32 42
%result = call i32 @inner1(i32* %ptr1, i32* %ptr2)
ret i32 %result
}
define i32 @inner1(i32* %begin, i32* %end) {
%begin.i = ptrtoint i32* %begin to i32
%end.i = ptrtoint i32* %end to i32
%distance = sub i32 %end.i, %begin.i
%icmp = icmp sle i32 %distance, 42
br i1 %icmp, label %then, label %else
then:
ret i32 3
else:
%t = load i32* %begin
ret i32 %t
}
define i32 @outer2(i32* %ptr) {
; Test that an inbounds GEP disables this -- it isn't safe in general as
; wrapping changes the behavior of lessthan and greaterthan comparisions.
; CHECK: @outer2
; CHECK: call i32 @inner2
; CHECK: ret i32
%ptr1 = getelementptr i32* %ptr, i32 0
%ptr2 = getelementptr i32* %ptr, i32 42
%result = call i32 @inner2(i32* %ptr1, i32* %ptr2)
ret i32 %result
}
define i32 @inner2(i32* %begin, i32* %end) {
%begin.i = ptrtoint i32* %begin to i32
%end.i = ptrtoint i32* %end to i32
%distance = sub i32 %end.i, %begin.i
%icmp = icmp sle i32 %distance, 42
br i1 %icmp, label %then, label %else
then:
ret i32 3
else:
%t = load i32* %begin
ret i32 %t
}