RPCSX/llvm
1
0
Fork 0
mirror of https://github.com/RPCSX/llvm.git synced 2024-11-28 14:10:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make use of vector load and store operations to implement memcpy, memmove, and memset. Currently only X86 target is taking advantage of these.

Browse Source 
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@51140 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Evan Cheng 2008-05-15 08:39:06 +00:00
parent 38503d4046
commit f0df03134e
5 changed files with 207 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/llvm/CodeGen/MachineFrameInfo.h
View File

@ -210,6 +210,13 @@ public:
return Objects[ObjectIdx+NumFixedObjects].Alignment;
}
/// setObjectAlignment - Change the alignment of the spcified stack object...
void setObjectAlignment(int ObjectIdx, unsigned Align) {
assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
"Invalid Object Idx!");
Objects[ObjectIdx+NumFixedObjects].Alignment = Align;
}
/// getObjectOffset - Return the assigned stack offset of the specified object
/// from the incoming stack pointer.
///

9
include/llvm/Target/TargetLowering.h
View File

@ -510,6 +510,15 @@ public:
bool allowsUnalignedMemoryAccesses() const {
return allowUnalignedMemoryAccesses;
}
/// getOptimalMemOpType - Returns the target specific optimal type for load
/// store operations as result of memset, memcpy, and memmove lowering.
/// It returns MVT::iAny if SelectionDAG should be responsible for
/// determining it.
virtual MVT::ValueType getOptimalMemOpType(uint64_t Size, unsigned Align,
bool isSrcConst, bool isSrcStr) const {
return MVT::iAny;
}
/// usesUnderscoreSetJmp - Determine if we should use _setjmp or setjmp
/// to implement llvm.setjmp.

231
lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View File

@ -2505,41 +2505,42 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
/// operand.
static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
SelectionDAG &DAG) {
MVT::ValueType CurVT = VT;
unsigned NumBits = MVT::isVector(VT) ?
MVT::getSizeInBits(MVT::getVectorElementType(VT)) : MVT::getSizeInBits(VT);
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
uint64_t Val = C->getValue() & 255;
APInt Val = APInt(NumBits, C->getValue() & 255);
unsigned Shift = 8;
while (CurVT != MVT::i8) {
for (unsigned i = NumBits; i > 8; i >>= 1) {
Val = (Val << Shift) | Val;
Shift <<= 1;
CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
}
return DAG.getConstant(Val, VT);
} else {
Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
unsigned Shift = 8;
while (CurVT != MVT::i8) {
Value =
DAG.getNode(ISD::OR, VT,
DAG.getNode(ISD::SHL, VT, Value,
DAG.getConstant(Shift, MVT::i8)), Value);
Shift <<= 1;
CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
}
return Value;
if (MVT::isInteger(VT))
return DAG.getConstant(Val, VT);
return DAG.getConstantFP(APFloat(Val), VT);
}
Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
unsigned Shift = 8;
for (unsigned i = NumBits; i > 8; i >>= 1) {
Value = DAG.getNode(ISD::OR, VT,
DAG.getNode(ISD::SHL, VT, Value,
DAG.getConstant(Shift, MVT::i8)), Value);
Shift <<= 1;
}
return Value;
}
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
/// used when a memcpy is turned into a memset when the source is a constant
/// string ptr.
static SDOperand getMemsetStringVal(MVT::ValueType VT,
SelectionDAG &DAG,
static SDOperand getMemsetStringVal(MVT::ValueType VT, SelectionDAG &DAG,
const TargetLowering &TLI,
std::string &Str, unsigned Offset) {
assert(!MVT::isVector(VT) && "Can't handle vector type here!");
unsigned NumBits = MVT::getSizeInBits(VT);
unsigned MSB = NumBits / 8;
uint64_t Val = 0;
unsigned MSB = MVT::getSizeInBits(VT) / 8;
if (TLI.isLittleEndian())
Offset = Offset + MSB - 1;
for (unsigned i = 0; i != MSB; ++i) {
@ -2550,56 +2551,119 @@ static SDOperand getMemsetStringVal(MVT::ValueType VT,
}
/// getMemBasePlusOffset - Returns base and offset node for the
///
static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
SelectionDAG &DAG) {
MVT::ValueType VT = Base.getValueType();
return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
}
/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
/// to replace the memset / memcpy is below the threshold. It also returns the
/// types of the sequence of memory ops to perform memset / memcpy.
static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
unsigned Limit, uint64_t Size,
unsigned Align,
const TargetLowering &TLI) {
MVT::ValueType VT;
/// isMemSrcFromString - Returns true if memcpy source is a string constant.
///
static bool isMemSrcFromString(SDOperand Src, std::string &Str,
uint64_t &SrcOff) {
unsigned SrcDelta = 0;
GlobalAddressSDNode *G = NULL;
if (Src.getOpcode() == ISD::GlobalAddress)
G = cast<GlobalAddressSDNode>(Src);
else if (Src.getOpcode() == ISD::ADD &&
Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
Src.getOperand(1).getOpcode() == ISD::Constant) {
G = cast<GlobalAddressSDNode>(Src.getOperand(0));
SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
}
if (!G)
return false;
if (TLI.allowsUnalignedMemoryAccesses()) {
VT = MVT::i64;
} else {
switch (Align & 7) {
case 0:
VT = MVT::i64;
break;
case 4:
VT = MVT::i32;
break;
case 2:
VT = MVT::i16;
break;
default:
VT = MVT::i8;
break;
GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
if (GV && GV->isConstant()) {
Str = GV->getStringValue(false);
if (!Str.empty()) {
SrcOff += SrcDelta;
return true;
}
}
MVT::ValueType LVT = MVT::i64;
while (!TLI.isTypeLegal(LVT))
LVT = (MVT::ValueType)((unsigned)LVT - 1);
assert(MVT::isInteger(LVT));
return false;
}
if (VT > LVT)
VT = LVT;
/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
/// to replace the memset / memcpy is below the threshold. It also returns the
/// types of the sequence of memory ops to perform memset / memcpy.
static
bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
SDOperand Dst, SDOperand Src,
unsigned Limit, uint64_t Size, unsigned &Align,
SelectionDAG &DAG,
const TargetLowering &TLI) {
bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses();
std::string Str;
uint64_t SrcOff = 0;
bool isSrcStr = isMemSrcFromString(Src, Str, SrcOff);
bool isSrcConst = isa<ConstantSDNode>(Src);
MVT::ValueType VT= TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr);
if (VT != MVT::iAny) {
unsigned NewAlign = (unsigned)
TLI.getTargetData()->getABITypeAlignment(MVT::getTypeForValueType(VT));
// If source is a string constant, this will require an unaligned load.
if (NewAlign > Align && (isSrcConst || AllowUnalign)) {
if (Dst.getOpcode() != ISD::FrameIndex) {
// Can't change destination alignment. It requires a unaligned store.
if (AllowUnalign)
VT = MVT::iAny;
} else {
int FI = cast<FrameIndexSDNode>(Dst)->getIndex();
MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
if (MFI->isFixedObjectIndex(FI)) {
// Can't change destination alignment. It requires a unaligned store.
if (AllowUnalign)
VT = MVT::iAny;
} else {
// Give the stack frame object a larger alignment.
MFI->setObjectAlignment(FI, NewAlign);
Align = NewAlign;
}
}
}
}
if (VT == MVT::iAny) {
if (AllowUnalign) {
VT = MVT::i64;
} else {
switch (Align & 7) {
case 0: VT = MVT::i64; break;
case 4: VT = MVT::i32; break;
case 2: VT = MVT::i16; break;
default: VT = MVT::i8; break;
}
}
MVT::ValueType LVT = MVT::i64;
while (!TLI.isTypeLegal(LVT))
LVT = (MVT::ValueType)((unsigned)LVT - 1);
assert(MVT::isInteger(LVT));
if (VT > LVT)
VT = LVT;
}
unsigned NumMemOps = 0;
while (Size != 0) {
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
while (VTSize > Size) {
VT = (MVT::ValueType)((unsigned)VT - 1);
VTSize >>= 1;
// For now, only use non-vector load / store's for the left-over pieces.
if (MVT::isVector(VT)) {
VT = MVT::i64;
while (!TLI.isTypeLegal(VT))
VT = (MVT::ValueType)((unsigned)VT - 1);
VTSize = MVT::getSizeInBits(VT) / 8;
} else {
VT = (MVT::ValueType)((unsigned)VT - 1);
VTSize >>= 1;
}
}
assert(MVT::isInteger(VT));
if (++NumMemOps > Limit)
return false;
@ -2613,8 +2677,7 @@ static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
SDOperand Chain, SDOperand Dst,
SDOperand Src, uint64_t Size,
unsigned Align,
bool AlwaysInline,
unsigned Align, bool AlwaysInline,
const Value *DstSV, uint64_t DstSVOff,
const Value *SrcSV, uint64_t SrcSVOff){
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@ -2625,56 +2688,38 @@ static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
uint64_t Limit = -1;
if (!AlwaysInline)
Limit = TLI.getMaxStoresPerMemcpy();
if (!MeetsMaxMemopRequirement(MemOps, Limit, Size, Align, TLI))
unsigned DstAlign = Align; // Destination alignment can change.
if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
DAG, TLI))
return SDOperand();
SmallVector<SDOperand, 8> OutChains;
unsigned NumMemOps = MemOps.size();
unsigned SrcDelta = 0;
GlobalAddressSDNode *G = NULL;
std::string Str;
bool CopyFromStr = false;
uint64_t SrcOff = 0, DstOff = 0;
bool CopyFromStr = isMemSrcFromString(Src, Str, SrcOff);
if (Src.getOpcode() == ISD::GlobalAddress)
G = cast<GlobalAddressSDNode>(Src);
else if (Src.getOpcode() == ISD::ADD &&
Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
Src.getOperand(1).getOpcode() == ISD::Constant) {
G = cast<GlobalAddressSDNode>(Src.getOperand(0));
SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
}
if (G) {
GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
if (GV && GV->isConstant()) {
Str = GV->getStringValue(false);
if (!Str.empty()) {
CopyFromStr = true;
SrcOff += SrcDelta;
}
}
}
SmallVector<SDOperand, 8> OutChains;
unsigned NumMemOps = MemOps.size();
for (unsigned i = 0; i < NumMemOps; i++) {
MVT::ValueType VT = MemOps[i];
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
SDOperand Value, Store;
if (CopyFromStr) {
if (CopyFromStr && !MVT::isVector(VT)) {
// It's unlikely a store of a vector immediate can be done in a single
// instruction. It would require a load from a constantpool first.
// FIXME: Handle cases where store of vector immediate is done in a
// single instruction.
Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
Store =
DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
DstSV, DstSVOff + DstOff);
Store = DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
DstSV, DstSVOff + DstOff);
} else {
Value = DAG.getLoad(VT, Chain,
getMemBasePlusOffset(Src, SrcOff, DAG),
SrcSV, SrcSVOff + SrcOff, false, Align);
Store =
DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
DstSV, DstSVOff + DstOff, false, Align);
Store = DAG.getStore(Chain, Value,
getMemBasePlusOffset(Dst, DstOff, DAG),
DstSV, DstSVOff + DstOff, false, DstAlign);
}
OutChains.push_back(Store);
SrcOff += VTSize;
@ -2695,8 +2740,8 @@ static SDOperand getMemsetStores(SelectionDAG &DAG,
// Expand memset to a series of load/store ops if the size operand
// falls below a certain threshold.
std::vector<MVT::ValueType> MemOps;
if (!MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(),
Size, Align, TLI))
if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, TLI.getMaxStoresPerMemset(),
Size, Align, DAG, TLI))
return SDOperand();
SmallVector<SDOperand, 8> OutChains;

61
lib/Target/X86/X86ISelLowering.cpp
View File

@ -787,6 +787,23 @@ unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
return Align;
}
/// getOptimalMemOpType - Returns the target specific optimal type for load
/// store operations as result of memset, memcpy, and memmove lowering.
/// It returns MVT::iAny if SelectionDAG should be responsible for
/// determining it.
MVT::ValueType
X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align,
bool isSrcConst, bool isSrcStr) const {
if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16)
return MVT::v4i32;
if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16)
return MVT::v4f32;
if (Subtarget->is64Bit() && Size >= 8)
return MVT::i64;
return MVT::i32;
}
/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC
/// jumptable.
SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table,
@ -2738,17 +2755,23 @@ static bool isZeroShuffle(SDNode *N) {
/// getZeroVector - Returns a vector of specified type with all zero elements.
///
static SDOperand getZeroVector(MVT::ValueType VT, SelectionDAG &DAG) {
static SDOperand getZeroVector(MVT::ValueType VT, bool HasSSE2,
SelectionDAG &DAG) {
assert(MVT::isVector(VT) && "Expected a vector type");
// Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest
// type. This ensures they get CSE'd.
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
SDOperand Vec;
if (MVT::getSizeInBits(VT) == 64) // MMX
if (MVT::getSizeInBits(VT) == 64) { // MMX
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst);
else // SSE
} else if (HasSSE2) { // SSE2
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst);
} else { // SSE1
SDOperand Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst);
}
return DAG.getNode(ISD::BIT_CONVERT, VT, Vec);
}
@ -2866,7 +2889,7 @@ static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) {
V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask);
NumElems >>= 1;
}
Mask = getZeroVector(MVT::v4i32, DAG);
Mask = getZeroVector(MVT::v4i32, true, DAG);
}
V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1);
@ -2880,9 +2903,11 @@ static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) {
/// element of V2 is swizzled into the zero/undef vector, landing at element
/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx,
bool isZero, SelectionDAG &DAG) {
bool isZero, bool HasSSE2,
SelectionDAG &DAG) {
MVT::ValueType VT = V2.getValueType();
SDOperand V1 = isZero ? getZeroVector(VT, DAG) : DAG.getNode(ISD::UNDEF, VT);
SDOperand V1 = isZero
? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT);
unsigned NumElems = MVT::getVectorNumElements(V2.getValueType());
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
MVT::ValueType EVT = MVT::getVectorElementType(MaskVT);
@ -2911,7 +2936,7 @@ static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros,
bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
if (ThisIsNonZero && First) {
if (NumZero)
V = getZeroVector(MVT::v8i16, DAG);
V = getZeroVector(MVT::v8i16, true, DAG);
else
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
First = false;
@ -2956,7 +2981,7 @@ static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros,
if (isNonZero) {
if (First) {
if (NumZero)
V = getZeroVector(MVT::v8i16, DAG);
V = getZeroVector(MVT::v8i16, true, DAG);
else
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
First = false;
@ -2981,7 +3006,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
if (ISD::isBuildVectorAllOnes(Op.Val))
return getOnesVector(Op.getValueType(), DAG);
return getZeroVector(Op.getValueType(), DAG);
return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG);
}
MVT::ValueType VT = Op.getValueType();
@ -3036,7 +3061,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
// convert it to a vector with movd (S2V+shuffle to zero extend).
Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item);
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item);
Item = getShuffleVectorZeroOrUndef(Item, 0, true, DAG);
Item = getShuffleVectorZeroOrUndef(Item, 0, true,
Subtarget->hasSSE2(), DAG);
// Now we have our 32-bit value zero extended in the low element of
// a vector. If Idx != 0, swizzle it into place.
@ -3061,7 +3087,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
(EVT != MVT::i64 || Subtarget->is64Bit())) {
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
// Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, DAG);
return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
Subtarget->hasSSE2(), DAG);
}
if (IsAllConstants) // Otherwise, it's better to do a constpool load.
@ -3076,7 +3103,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
// Turn it into a shuffle of zero and zero-extended scalar to vector.
Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, DAG);
Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
Subtarget->hasSSE2(), DAG);
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
MVT::ValueType MaskEVT = MVT::getVectorElementType(MaskVT);
SmallVector<SDOperand, 8> MaskVec;
@ -3105,7 +3133,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
unsigned Idx = CountTrailingZeros_32(NonZeros);
SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT,
Op.getOperand(Idx));
return getShuffleVectorZeroOrUndef(V2, Idx, true, DAG);
return getShuffleVectorZeroOrUndef(V2, Idx, true,
Subtarget->hasSSE2(), DAG);
}
return SDOperand();
}
@ -3130,7 +3159,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
for (unsigned i = 0; i < 4; ++i) {
bool isZero = !(NonZeros & (1 << i));
if (isZero)
V[i] = getZeroVector(VT, DAG);
V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG);
else
V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
}
@ -3542,7 +3571,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) {
return DAG.getNode(ISD::UNDEF, VT);
if (isZeroShuffle(Op.Val))
return getZeroVector(VT, DAG);
return getZeroVector(VT, Subtarget->hasSSE2(), DAG);
if (isIdentityMask(PermMask.Val))
return V1;

8
lib/Target/X86/X86ISelLowering.h
View File

@ -336,6 +336,14 @@ namespace llvm {
/// that contains are placed at 16-byte boundaries while the rest are at
/// 4-byte boundaries.
virtual unsigned getByValTypeAlignment(const Type *Ty) const;
/// getOptimalMemOpType - Returns the target specific optimal type for load
/// store operations as result of memset, memcpy, and memmove lowering.
/// It returns MVT::iAny if SelectionDAG should be responsible for
/// determining it.
virtual
MVT::ValueType getOptimalMemOpType(uint64_t Size, unsigned Align,
bool isSrcConst, bool isSrcStr) const;
/// LowerOperation - Provide custom lowering hooks for some operations.
///