From 0285c1e9bbdc12b0cb6262ae9ff43a532d9b3a60 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Mon, 28 Apr 2008 17:15:20 +0000 Subject: [PATCH] Fix the SVOffset values for loads and stores produced by memcpy/memset expansion. It was a bug for the SVOffset value to be used in the actual address calculations. llvm-svn: 50359 --- include/llvm/CodeGen/SelectionDAG.h | 10 +++--- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 38 ++++++++++++----------- lib/Target/ARM/ARMISelLowering.cpp | 13 ++++---- lib/Target/ARM/ARMISelLowering.h | 4 +-- lib/Target/X86/X86ISelLowering.cpp | 11 ++++--- lib/Target/X86/X86ISelLowering.h | 6 ++-- 6 files changed, 43 insertions(+), 39 deletions(-) diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 0d0e4d374cb..e17e32eaf14 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -326,17 +326,17 @@ public: SDOperand getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff); + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff); SDOperand getMemmove(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff); + const Value *DstSV, uint64_t DstOSVff, + const Value *SrcSV, uint64_t SrcSVOff); SDOperand getMemset(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff); + const Value *DstSV, uint64_t DstSVOff); /// getSetCC - Helper function to make it easier to build SetCC's if you just /// have an ISD::CondCode instead of an SDOperand. diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index bfc35068994..dfd422a4a93 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2590,8 +2590,8 @@ static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG, SDOperand Src, uint64_t Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff) { + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff){ const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // Expand memcpy to a series of store ops if the size operand falls below @@ -2610,6 +2610,7 @@ static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG, GlobalAddressSDNode *G = NULL; std::string Str; bool CopyFromStr = false; + uint64_t SrcOff = 0, DstOff = 0; if (Src.getOpcode() == ISD::GlobalAddress) G = cast(Src); @@ -2640,15 +2641,15 @@ static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG, Store = DAG.getStore(Chain, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstOff); + DstSV, DstSVOff + DstOff); } else { Value = DAG.getLoad(VT, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), - SrcSV, SrcOff, false, Align); + SrcSV, SrcSVOff + SrcOff, false, Align); Store = DAG.getStore(Chain, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstOff, false, Align); + DstSV, DstSVOff + DstOff, false, Align); } OutChains.push_back(Store); SrcOff += VTSize; @@ -2663,7 +2664,7 @@ static SDOperand getMemsetStores(SelectionDAG &DAG, SDOperand Chain, SDOperand Dst, SDOperand Src, uint64_t Size, unsigned Align, - const Value *DstSV, uint64_t DstOff) { + const Value *DstSV, uint64_t DstSVOff) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // Expand memset to a series of load/store ops if the size operand @@ -2674,6 +2675,7 @@ static SDOperand getMemsetStores(SelectionDAG &DAG, return SDOperand(); SmallVector OutChains; + uint64_t DstOff = 0; unsigned NumMemOps = MemOps.size(); for (unsigned i = 0; i < NumMemOps; i++) { @@ -2682,7 +2684,7 @@ static SDOperand getMemsetStores(SelectionDAG &DAG, SDOperand Value = getMemsetValue(Src, VT, DAG); SDOperand Store = DAG.getStore(Chain, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstOff); + DstSV, DstSVOff + DstOff); OutChains.push_back(Store); DstOff += VTSize; } @@ -2694,8 +2696,8 @@ static SDOperand getMemsetStores(SelectionDAG &DAG, SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff) { + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff) { // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. @@ -2707,7 +2709,7 @@ SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Result = getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(), - Align, false, DstSV, DstOff, SrcSV, SrcOff); + Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.Val) return Result; } @@ -2717,7 +2719,7 @@ SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Result = TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align, AlwaysInline, - DstSV, DstOff, SrcSV, SrcOff); + DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.Val) return Result; @@ -2727,7 +2729,7 @@ SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, assert(ConstantSize && "AlwaysInline requires a constant size!"); return getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align, true, - DstSV, DstOff, SrcSV, SrcOff); + DstSV, DstSVOff, SrcSV, SrcSVOff); } // Emit a library call. @@ -2748,8 +2750,8 @@ SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff) { + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff) { // TODO: Optimize small memmove cases with simple loads and stores, // ensuring that all loads precede all stores. This can cause severe @@ -2759,7 +2761,7 @@ SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst, // code. If the target chooses to do this, this is the next best. SDOperand Result = TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align, - DstSV, DstOff, SrcSV, SrcOff); + DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.Val) return Result; @@ -2781,7 +2783,7 @@ SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst, SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff) { + const Value *DstSV, uint64_t DstSVOff) { // Check to see if we should lower the memset to stores first. // For cases within the target-specified limits, this is the best choice. @@ -2793,7 +2795,7 @@ SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst, SDOperand Result = getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align, - DstSV, DstOff); + DstSV, DstSVOff); if (Result.Val) return Result; } @@ -2802,7 +2804,7 @@ SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst, // code. If the target chooses to do this, this is the next best. SDOperand Result = TLI.EmitTargetCodeForMemset(*this, Chain, Dst, Src, Size, Align, - DstSV, DstOff); + DstSV, DstSVOff); if (Result.Val) return Result; diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index 6a581f715fd..dc76b7b333f 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -1247,8 +1247,8 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff){ + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff){ // Do repeated 4-byte loads and stores. To be improved. // This requires 4-byte alignment. if ((Align & 3) != 0) @@ -1271,6 +1271,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, const unsigned MAX_LOADS_IN_LDM = 6; SDOperand TFOps[MAX_LOADS_IN_LDM]; SDOperand Loads[MAX_LOADS_IN_LDM]; + uint64_t SrcOff = 0, DstOff = 0; // Emit up to MAX_LOADS_IN_LDM loads, then a TokenFactor barrier, then the // same number of stores. The loads and stores will get combined into @@ -1281,7 +1282,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, Loads[i] = DAG.getLoad(VT, Chain, DAG.getNode(ISD::ADD, MVT::i32, Src, DAG.getConstant(SrcOff, MVT::i32)), - SrcSV, SrcOff); + SrcSV, SrcSVOff + SrcOff); TFOps[i] = Loads[i].getValue(1); SrcOff += VTSize; } @@ -1292,7 +1293,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, TFOps[i] = DAG.getStore(Chain, Loads[i], DAG.getNode(ISD::ADD, MVT::i32, Dst, DAG.getConstant(DstOff, MVT::i32)), - DstSV, DstOff); + DstSV, DstSVOff + DstOff); DstOff += VTSize; } Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &TFOps[0], i); @@ -1318,7 +1319,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, Loads[i] = DAG.getLoad(VT, Chain, DAG.getNode(ISD::ADD, MVT::i32, Src, DAG.getConstant(SrcOff, MVT::i32)), - SrcSV, SrcOff); + SrcSV, SrcSVOff + SrcOff); TFOps[i] = Loads[i].getValue(1); ++i; SrcOff += VTSize; @@ -1340,7 +1341,7 @@ ARMTargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, TFOps[i] = DAG.getStore(Chain, Loads[i], DAG.getNode(ISD::ADD, MVT::i32, Dst, DAG.getConstant(DstOff, MVT::i32)), - DstSV, DstOff); + DstSV, DstSVOff + DstOff); ++i; DstOff += VTSize; BytesLeft -= VTSize; diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h index 13f5c083753..ce6d5fe449a 100644 --- a/lib/Target/ARM/ARMISelLowering.h +++ b/lib/Target/ARM/ARMISelLowering.h @@ -149,8 +149,8 @@ namespace llvm { SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff); + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff); }; } diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index c0ba4d1b9ac..799ba007fa1 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -4763,7 +4763,7 @@ X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff) { + const Value *DstSV, uint64_t DstSVOff) { ConstantSDNode *ConstantSize = dyn_cast(Size); /// If not DWORD aligned or size is more than the threshold, call the library. @@ -4890,7 +4890,7 @@ X86TargetLowering::EmitTargetCodeForMemset(SelectionDAG &DAG, DAG.getConstant(Offset, AddrVT)), Src, DAG.getConstant(BytesLeft, SizeVT), - Align, DstSV, 0); + Align, DstSV, DstSVOff + Offset); } // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain. @@ -4903,8 +4903,8 @@ X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff){ + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff){ // This requires the copy size to be a constant, preferrably // within a subtarget-specific limit. @@ -4964,7 +4964,8 @@ X86TargetLowering::EmitTargetCodeForMemcpy(SelectionDAG &DAG, DAG.getConstant(Offset, SrcVT)), DAG.getConstant(BytesLeft, SizeVT), Align, AlwaysInline, - DstSV, 0, SrcSV, 0)); + DstSV, DstSVOff + Offset, + SrcSV, SrcSVOff + Offset)); } return DAG.getNode(ISD::TokenFactor, MVT::Other, &Results[0], Results.size()); diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index 6b02f33aadc..987cf73102f 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -538,14 +538,14 @@ namespace llvm { SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, - const Value *DstSV, uint64_t DstOff); + const Value *DstSV, uint64_t DstSVOff); SDOperand EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDOperand Chain, SDOperand Dst, SDOperand Src, SDOperand Size, unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstOff, - const Value *SrcSV, uint64_t SrcOff); + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff); }; }