RPCSX/llvm
1
0
Fork 0
mirror of https://github.com/RPCSX/llvm.git synced 2024-12-02 16:56:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

SelectionDAG: Store SDNode operands in an ArrayRecycler

Browse Source 
Currently some SDNode operands are malloc'd, some are stored inline in
subclasses of SDNode, and some are thrown into a BumpPtrAllocator.
This scheme is complex, inconsistent, and makes refactoring SDNodes
fairly difficult.

Instead, we can allocate all of the operands using an ArrayRecycler
that wraps a BumpPtrAllocator. This keeps the cache locality when
iterating operands, improves locality when iterating SDNodes without
looking at operands, and vastly simplifies the ownership semantics.

It also means we stop overallocating SDNodes by 2-3x and will make it
simpler to fix the rampant undefined behaviour we have in how we
mutate SDNodes from one kind to another (See llvm.org/pr26808).

This is NFC other than the changes in memory behaviour, and I ran some
LNT tests to make sure this didn't hurt compile time. Not many tests
changed: there were a couple of 1-2% regressions reported, but there
were more improvements (of up to 4%) than regressions.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@262886 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Justin Bogner 2016-03-08 00:39:51 +00:00
parent 4ae6fddb70
commit 54124b63dc
3 changed files with 213 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
include/llvm/CodeGen/SelectionDAG.h
View File

@ -23,6 +23,7 @@
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/Support/ArrayRecycler.h"
#include "llvm/Support/RecyclingAllocator.h"
#include "llvm/Target/TargetMachine.h"
#include <cassert>
@ -208,6 +209,7 @@ class SelectionDAG {
/// Pool allocation for machine-opcode SDNode operands.
BumpPtrAllocator OperandAllocator;
ArrayRecycler<SDUse> OperandRecycler;
/// Pool allocation for misc. objects that are created once per SelectionDAG.
BumpPtrAllocator Allocator;
@ -274,6 +276,30 @@ private:
SDNodeT(std::forward<ArgTypes>(Args)...);
}
void createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
assert(!Node->OperandList && "Node already has operands");
SDUse *Ops = OperandRecycler.allocate(
ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
for (unsigned I = 0; I != Vals.size(); ++I) {
Ops[I].setUser(Node);
Ops[I].setInitial(Vals[I]);
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;
checkForCycles(Node);
}
void removeOperands(SDNode *Node) {
if (!Node->OperandList)
return;
OperandRecycler.deallocate(
ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
Node->OperandList);
Node->NumOperands = 0;
Node->OperandList = nullptr;
}
void operator=(const SelectionDAG&) = delete;
SelectionDAG(const SelectionDAG&) = delete;
@ -1316,9 +1342,8 @@ private:
void allnodes_clear();
BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
SDValue N1, SDValue N2,
const SDNodeFlags *Flags = nullptr);
SDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
SDValue N2, const SDNodeFlags *Flags = nullptr);
/// Look up the node specified by ID in CSEMap. If it exists, return it. If
/// not, return the insertion token that will make insertion faster. This

313
include/llvm/CodeGen/SelectionDAGNodes.h
View File

@ -44,7 +44,7 @@ class GlobalValue;
class MachineBasicBlock;
class MachineConstantPoolValue;
class SDNode;
class BinaryWithFlagsSDNode;
class HandleSDNode;
class Value;
class MCSymbol;
template <typename T> struct DenseMapInfo;
@ -280,6 +280,8 @@ public:
private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;
void setUser(SDNode *p) { User = p; }
@ -394,10 +396,6 @@ private:
/// The operation that this node performs.
int16_t NodeType;
/// This is true if OperandList was new[]'d. If true,
/// then they will be delete[]'d when the node is destroyed.
uint16_t OperandsNeedDelete : 1;
/// This tracks whether this node has one or more dbg_value
/// nodes corresponding to it.
uint16_t HasDebugValue : 1;
@ -406,7 +404,7 @@ protected:
/// This member is defined by this class, but is not used for
/// anything. Subclasses can use it to hold whatever state they find useful.
/// This field is initialized to zero by the ctor.
uint16_t SubclassData : 14;
uint16_t SubclassData : 15;
private:
/// Unique id per SDNode in the DAG.
@ -440,6 +438,8 @@ private:
friend class SelectionDAG;
friend struct ilist_traits<SDNode>;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;
public:
/// Unique and persistent id per SDNode in the DAG.
@ -792,101 +792,20 @@ protected:
return Ret;
}
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops)
: NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false),
SubclassData(0), NodeId(-1),
OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()),
NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) {
assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
assert(NumOperands == Ops.size() &&
"NumOperands wasn't wide enough for its operands!");
assert(NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!");
for (unsigned i = 0; i != Ops.size(); ++i) {
assert(OperandList && "no operands available");
OperandList[i].setUser(this);
OperandList[i].setInitial(Ops[i]);
}
checkForCycles(this);
}
/// This constructor adds no operands itself; operands can be
/// set later with InitOperands.
/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
: NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs),
IROrder(Order), debugLoc(std::move(dl)) {
: NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1),
OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr),
NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order),
debugLoc(std::move(dl)) {
assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
assert(NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!");
}
/// Initialize the operands list of this with 1 operand.
void InitOperands(SDUse *Ops, const SDValue &Op0) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
NumOperands = 1;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 2 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
NumOperands = 2;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 3 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
const SDValue &Op2) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
Ops[2].setUser(this);
Ops[2].setInitial(Op2);
NumOperands = 3;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 4 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
const SDValue &Op2, const SDValue &Op3) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
Ops[2].setUser(this);
Ops[2].setInitial(Op2);
Ops[3].setUser(this);
Ops[3].setInitial(Op3);
NumOperands = 4;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with N operands.
void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) {
for (unsigned i = 0; i != N; ++i) {
Ops[i].setUser(this);
Ops[i].setInitial(Vals[i]);
}
NumOperands = N;
assert(NumOperands == N &&
"NumOperands wasn't wide enough for its operands!");
OperandList = Ops;
checkForCycles(this);
}
/// Release the operands and set this node to have zero operands.
void DropOperands();
};
@ -1012,30 +931,6 @@ inline void SDUse::setNode(SDNode *N) {
if (N) N->addUse(*this);
}
/// This class is used for single-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class UnarySDNode : public SDNode {
SDUse Op;
public:
UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(&Op, X);
}
};
/// This class is used for two-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class BinarySDNode : public SDNode {
SDUse Ops[2];
public:
BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(Ops, X, Y);
}
};
/// Returns true if the opcode is a binary operation with flags.
static bool isBinOpWithFlags(unsigned Opcode) {
switch (Opcode) {
@ -1060,30 +955,17 @@ static bool isBinOpWithFlags(unsigned Opcode) {
/// This class is an extension of BinarySDNode
/// used from those opcodes that have associated extra flags.
class BinaryWithFlagsSDNode : public BinarySDNode {
class BinaryWithFlagsSDNode : public SDNode {
public:
SDNodeFlags Flags;
BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y, const SDNodeFlags &NodeFlags)
: BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {}
const SDNodeFlags &NodeFlags)
: SDNode(Opc, Order, dl, VTs), Flags(NodeFlags) {}
static bool classof(const SDNode *N) {
return isBinOpWithFlags(N->getOpcode());
}
};
/// This class is used for three-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class TernarySDNode : public SDNode {
SDUse Ops[3];
public:
TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y, SDValue Z)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(Ops, X, Y, Z);
}
};
/// This class is used to form a handle around another node that
/// is persistent and is updated across invocations of replaceAllUsesWith on its
/// operand. This node should be directly created by end-users and not added to
@ -1096,20 +978,28 @@ public:
// HandleSDNodes are never inserted into the DAG, so they won't be
// auto-numbered. Use ID 65535 as a sentinel.
PersistentId = 0xffff;
InitOperands(&Op, X);
// Manually set up the operand list. This node type is special in that it's
// always stack allocated and SelectionDAG does not manage its operands.
// TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
// be so special.
Op.setUser(this);
Op.setInitial(X);
NumOperands = 1;
OperandList = &Op;
}
~HandleSDNode();
const SDValue &getValue() const { return Op; }
};
class AddrSpaceCastSDNode : public UnarySDNode {
class AddrSpaceCastSDNode : public SDNode {
private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;
public:
AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X,
unsigned SrcAS, unsigned DestAS);
AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, unsigned SrcAS,
unsigned DestAS);
unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }
@ -1133,9 +1023,6 @@ public:
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT MemoryVT, MachineMemOperand *MMO);
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT MemoryVT, MachineMemOperand *MMO);
bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }
@ -1238,8 +1125,6 @@ public:
/// This is an SDNode representing atomic operations.
class AtomicSDNode : public MemSDNode {
SDUse Ops[4];
/// For cmpxchg instructions, the ordering requirements when a store does not
/// occur.
AtomicOrdering FailureOrdering;
@ -1265,16 +1150,12 @@ class AtomicSDNode : public MemSDNode {
}
public:
AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT,
const SDValue* AllOps, SDUse *DynOps, unsigned NumOps,
MachineMemOperand *MMO,
AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
EVT MemVT, MachineMemOperand *MMO,
AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
SynchronizationScope SynchScope)
: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
assert((DynOps || NumOps <= array_lengthof(Ops)) &&
"Too many ops for internal storage!");
InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps);
}
const SDValue &getBasePtr() const { return getOperand(1); }
@ -1322,9 +1203,8 @@ public:
class MemIntrinsicSDNode : public MemSDNode {
public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT MemoryVT,
MachineMemOperand *MMO)
: MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) {
EVT MemoryVT, MachineMemOperand *MMO)
: MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
SubclassData |= 1u << 13;
}
@ -1347,20 +1227,15 @@ public:
/// An index of -1 is treated as undef, such that the code generator may put
/// any value in the corresponding element of the result.
class ShuffleVectorSDNode : public SDNode {
SDUse Ops[2];
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;
protected:
friend class SelectionDAG;
ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1,
SDValue N2, const int *M)
: SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {
InitOperands(Ops, N1, N2);
}
public:
ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, const int *M)
: SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
public:
ArrayRef<int> getMask() const {
EVT VT = getValueType(0);
return makeArrayRef(Mask, VT.getVectorNumElements());
@ -1791,13 +1666,11 @@ public:
};
class EHLabelSDNode : public SDNode {
SDUse Chain;
MCSymbol *Label;
friend class SelectionDAG;
EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L)
: SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {
InitOperands(&Chain, ch);
}
EHLabelSDNode(unsigned Order, DebugLoc dl, MCSymbol *L)
: SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {}
public:
MCSymbol *getLabel() const { return Label; }
@ -1863,11 +1736,10 @@ class CvtRndSatSDNode : public SDNode {
ISD::CvtCode CvtCode;
friend class SelectionDAG;
explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl,
ArrayRef<SDValue> Ops, ISD::CvtCode Code)
: SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops),
CvtCode(Code) {
assert(Ops.size() == 5 && "wrong number of operations");
ISD::CvtCode Code)
: SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT)), CvtCode(Code) {
}
public:
ISD::CvtCode getCvtCode() const { return CvtCode; }
@ -1896,24 +1768,13 @@ public:
/// Base class for LoadSDNode and StoreSDNode
class LSBaseSDNode : public MemSDNode {
//! Operand array for load and store
/*!
\note Moving this array to the base class captures more
common functionality shared between LoadSDNode and
StoreSDNode
*/
SDUse Ops[4];
public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
SDValue *Operands, unsigned numOperands,
SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
SubclassData |= AM << 2;
assert(getAddressingMode() == AM && "MemIndexedMode encoding error!");
InitOperands(Ops, Operands, numOperands);
assert((getOffset().getOpcode() == ISD::UNDEF || isIndexed()) &&
"Only indexed loads and stores have a non-undef offset operand");
}
const SDValue &getOffset() const {
@ -1941,10 +1802,9 @@ public:
/// This class is used to represent ISD::LOAD nodes.
class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;
LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs,
ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
MachineMemOperand *MMO)
: LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) {
LoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
: LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)ETy;
assert(getExtensionType() == ETy && "LoadExtType encoding error!");
assert(readMem() && "Load MachineMemOperand is not a load!");
@ -1969,11 +1829,9 @@ public:
/// This class is used to represent ISD::STORE nodes.
class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;
StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl,
SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
MachineMemOperand *MMO)
: LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4,
VTs, AM, MemVT, MMO) {
StoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
bool isTrunc, EVT MemVT, MachineMemOperand *MMO)
: LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc;
assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!");
assert(!readMem() && "Store MachineMemOperand is a load!");
@ -1997,16 +1855,11 @@ public:
/// This base class is used to represent MLOAD and MSTORE nodes
class MaskedLoadStoreSDNode : public MemSDNode {
// Operands
SDUse Ops[4];
public:
friend class SelectionDAG;
MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
SDValue *Operands, unsigned numOperands, SDVTList VTs,
EVT MemVT, MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
InitOperands(Ops, Operands, numOperands);
}
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
// In the both nodes address is Op1, mask is Op2:
// MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value
@ -2025,11 +1878,9 @@ public:
class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
public:
friend class SelectionDAG;
MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,
unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy,
EVT MemVT, MachineMemOperand *MMO)
: MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands,
VTs, MemVT, MMO) {
MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs,
ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
: MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
SubclassData |= (unsigned short)ETy;
}
@ -2047,12 +1898,10 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
public:
friend class SelectionDAG;
MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands,
unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT,
MachineMemOperand *MMO)
: MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands,
VTs, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc;
MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, bool isTrunc,
EVT MemVT, MachineMemOperand *MMO)
: MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc;
}
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
@ -2070,17 +1919,11 @@ public:
/// MGATHER and MSCATTER nodes
///
class MaskedGatherScatterSDNode : public MemSDNode {
// Operands
SDUse Ops[5];
public:
friend class SelectionDAG;
MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT,
MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
assert(Operands.size() == 5 && "Incompatible number of operands");
InitOperands(Ops, Operands.data(), Operands.size());
}
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value
@ -2102,19 +1945,9 @@ public:
class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
public:
friend class SelectionDAG;
MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands,
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT,
MMO) {
assert(getValue().getValueType() == getValueType(0) &&
"Incompatible type of the PassThru value in MaskedGatherSDNode");
assert(getMask().getValueType().getVectorNumElements() ==
getValueType(0).getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(getIndex().getValueType().getVectorNumElements() ==
getValueType(0).getVectorNumElements() &&
"Vector width mismatch between index and data");
}
MaskedGatherSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {}
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MGATHER;
@ -2127,17 +1960,9 @@ class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
public:
friend class SelectionDAG;
MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands,
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT,
MMO) {
assert(getMask().getValueType().getVectorNumElements() ==
getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(getIndex().getValueType().getVectorNumElements() ==
getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between index and data");
}
MaskedScatterSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {}
static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MSCATTER;
@ -2156,10 +1981,6 @@ private:
MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs)
: SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {}
/// Operands for this instruction, if they fit here. If
/// they don't, this field is unused.
SDUse LocalOperands[4];
/// Memory reference descriptions for this instruction.
mmo_iterator MemRefs;
mmo_iterator MemRefsEnd;
@ -2235,7 +2056,7 @@ template <> struct GraphTraits<SDNode*> {
};
/// The largest SDNode class.
typedef MaskedGatherScatterSDNode LargestSDNode;
typedef AtomicSDNode LargestSDNode;
/// The SDNode class with the greatest alignment requirement.
typedef GlobalAddressSDNode MostAlignedSDNode;

261
lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View File

@ -664,8 +664,8 @@ void SDDbgInfo::erase(const SDNode *Node) {
}
void SelectionDAG::DeallocateNode(SDNode *N) {
if (N->OperandsNeedDelete)
delete[] N->OperandList;
// If we have operands, deallocate them.
removeOperands(N);
// Set the opcode to DELETED_NODE to help catch bugs when node
// memory is reallocated.
@ -912,6 +912,7 @@ void SelectionDAG::init(MachineFunction &mf) {
SelectionDAG::~SelectionDAG() {
assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
allnodes_clear();
OperandRecycler.clear(OperandAllocator);
delete DbgInfo;
}
@ -925,24 +926,26 @@ void SelectionDAG::allnodes_clear() {
#endif
}
BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
SDVTList VTs, SDValue N1,
SDValue N2,
const SDNodeFlags *Flags) {
SDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
SDValue N1, SDValue N2,
const SDNodeFlags *Flags) {
SDValue Ops[] = {N1, N2};
if (isBinOpWithFlags(Opcode)) {
// If no flags were passed in, use a default flags object.
SDNodeFlags F;
if (Flags == nullptr)
Flags = &F;
auto *FN = newSDNode<BinaryWithFlagsSDNode>(
Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
auto *FN = newSDNode<BinaryWithFlagsSDNode>(Opcode, DL.getIROrder(),
DL.getDebugLoc(), VTs, *Flags);
createOperands(FN, Ops);
return FN;
}
auto *N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTs, N1, N2);
auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
return N;
}
@ -982,6 +985,7 @@ SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
void SelectionDAG::clear() {
allnodes_clear();
OperandRecycler.clear(OperandAllocator);
OperandAllocator.Reset();
CSEMap.clear();
@ -1638,7 +1642,9 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
dl.getDebugLoc(), N1, N2, MaskAlloc);
dl.getDebugLoc(), MaskAlloc);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -1671,8 +1677,10 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
auto *N = newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(),
Ops, Code);
auto *N =
newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), Code);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -1715,8 +1723,9 @@ SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0);
auto *N =
newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Root, Label);
auto *N = newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -1799,7 +1808,9 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
return SDValue(E, 0);
auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
VT, Ptr, SrcAS, DestAS);
VT, SrcAS, DestAS);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -3183,20 +3194,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
SDNode *N;
SDVTList VTs = getVTList(VT);
SDValue Ops[] = {Operand};
if (VT != MVT::Glue) { // Don't CSE flag producing nodes
FoldingSetNodeID ID;
SDValue Ops[1] = { Operand };
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
Operand);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else {
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
Operand);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
}
InsertNode(N);
@ -3895,7 +3906,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
}
// Memoize this node if possible.
BinarySDNode *N;
SDNode *N;
SDVTList VTs = getVTList(VT);
if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2};
@ -3909,7 +3920,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
}
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
CSEMap.InsertNode(N, IP);
} else {
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
@ -4000,20 +4010,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
// Memoize node if it doesn't produce a flag.
SDNode *N;
SDVTList VTs = getVTList(VT);
SDValue Ops[] = {N1, N2, N3};
if (VT != MVT::Glue) {
SDValue Ops[] = { N1, N2, N3 };
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
N1, N2, N3);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs,
N1, N2, N3);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
}
InsertNode(N);
@ -4805,18 +4815,11 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
return SDValue(E, 0);
}
// Allocate the operands array for the node out of the BumpPtrAllocator, since
// SDNode doesn't have access to it. This memory will be "leaked" when
// the node is deallocated, but recovered when the allocator is released.
// If the number of operands is less than 5 we use AtomicSDNode's internal
// storage.
unsigned NumOps = Ops.size();
SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
: nullptr;
auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, MemVT, MMO, SuccessOrdering,
FailureOrdering, SynchScope);
createOperands(N, Ops);
auto *N = newSDNode<AtomicSDNode>(
Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, MemVT, Ops.data(),
DynOps, NumOps, MMO, SuccessOrdering, FailureOrdering, SynchScope);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5009,11 +5012,14 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
}
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, Ops, MemVT, MMO);
CSEMap.InsertNode(N, IP);
VTList, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else {
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, Ops, MemVT, MMO);
VTList, MemVT, MMO);
createOperands(N, Ops);
}
InsertNode(N);
return SDValue(N, 0);
@ -5131,8 +5137,10 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
cast<LoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<LoadSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
AM, ExtType, MemVT, MMO);
auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
ExtType, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5238,8 +5246,10 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, false, VT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5306,8 +5316,10 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, true, SVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5330,9 +5342,11 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0);
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs,
AM, ST->isTruncatingStore(),
ST->getMemoryVT(), ST->getMemOperand());
auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
ST->isTruncatingStore(), ST->getMemoryVT(),
ST->getMemOperand());
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5358,8 +5372,10 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops,
4, VTs, ExtTy, MemVT, MMO);
auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
ExtTy, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5384,8 +5400,10 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops,
4, VTs, isTrunc, MemVT, MMO);
auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
isTrunc, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5395,6 +5413,7 @@ SDValue
SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) {
assert(Ops.size() == 5 && "Incompatible number of operands");
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
@ -5409,8 +5428,20 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
Ops, VTs, VT, MMO);
VTs, VT, MMO);
createOperands(N, Ops);
assert(N->getValue().getValueType() == N->getValueType(0) &&
"Incompatible type of the PassThru value in MaskedGatherSDNode");
assert(N->getMask().getValueType().getVectorNumElements() ==
N->getValueType(0).getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(N->getIndex().getValueType().getVectorNumElements() ==
N->getValueType(0).getVectorNumElements() &&
"Vector width mismatch between index and data");
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5419,6 +5450,8 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) {
assert(Ops.size() == 5 && "Incompatible number of operands");
FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
ID.AddInteger(VT.getRawBits());
@ -5432,7 +5465,16 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
return SDValue(E, 0);
}
auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
Ops, VTs, VT, MMO);
VTs, VT, MMO);
createOperands(N, Ops);
assert(N->getMask().getValueType().getVectorNumElements() ==
N->getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(N->getIndex().getValueType().getVectorNumElements() ==
N->getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between index and data");
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@ -5511,10 +5553,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
}
InsertNode(N);
@ -5556,7 +5601,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
// Memoize the node unless it returns a flag.
SDNode *N;
unsigned NumOps = Ops.size();
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
@ -5564,34 +5608,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0);
if (NumOps == 1) {
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0]);
} else if (NumOps == 2) {
N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1]);
} else if (NumOps == 3) {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1], Ops[2]);
} else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
Ops);
}
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else {
if (NumOps == 1) {
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0]);
} else if (NumOps == 2) {
N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1]);
} else if (NumOps == 3) {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1], Ops[2]);
} else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
Ops);
}
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
createOperands(N, Ops);
}
InsertNode(N);
return SDValue(N, 0);
@ -5977,7 +5999,6 @@ SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
/// deleting things.
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
SDVTList VTs, ArrayRef<SDValue> Ops) {
unsigned NumOps = Ops.size();
// If an identical node already exists, use it.
void *IP = nullptr;
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
@ -6006,36 +6027,13 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
DeadNodeSet.insert(Used);
}
if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
// Initialize the memory references information.
// For MachineNode, initialize the memory references information.
if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
MN->setMemRefs(nullptr, nullptr);
// If NumOps is larger than the # of operands we can have in a
// MachineSDNode, reallocate the operand list.
if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
if (MN->OperandsNeedDelete)
delete[] MN->OperandList;
if (NumOps > array_lengthof(MN->LocalOperands))
// We're creating a final node that will live unmorphed for the
// remainder of the current SelectionDAG iteration, so we can allocate
// the operands directly out of a pool with no recycling metadata.
MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
Ops.data(), NumOps);
else
MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
MN->OperandsNeedDelete = false;
} else
MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
} else {
// If NumOps is larger than the # of operands we currently have, reallocate
// the operand list.
if (NumOps > N->NumOperands) {
if (N->OperandsNeedDelete)
delete[] N->OperandList;
N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
N->OperandsNeedDelete = true;
} else
N->InitOperands(N->OperandList, Ops.data(), NumOps);
}
// Swap for an appropriately sized array from the recycler.
removeOperands(N);
createOperands(N, Ops);
// Delete any nodes that are still dead after adding the uses for the
// new operands.
@ -6178,16 +6176,14 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
MachineSDNode *
SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
ArrayRef<SDValue> OpsArray) {
ArrayRef<SDValue> Ops) {
bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
MachineSDNode *N;
void *IP = nullptr;
const SDValue *Ops = OpsArray.data();
unsigned NumOps = OpsArray.size();
if (DoCSE) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
AddNodeIDNode(ID, ~Opcode, VTs, Ops);
IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
@ -6196,17 +6192,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
// Allocate a new MachineSDNode.
N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
// Initialize the operands list.
if (NumOps > array_lengthof(N->LocalOperands))
// We're creating a final node that will live unmorphed for the
// remainder of the current SelectionDAG iteration, so we can allocate
// the operands directly out of a pool with no recycling metadata.
N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
Ops, NumOps);
else
N->InitOperands(N->LocalOperands, Ops, NumOps);
N->OperandsNeedDelete = false;
createOperands(N, Ops);
if (DoCSE)
CSEMap.InsertNode(N, IP);
@ -6739,10 +6725,9 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
}
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
SDValue X, unsigned SrcAS,
unsigned DestAS)
: UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
unsigned SrcAS, unsigned DestAS)
: SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT memvt, MachineMemOperand *mmo)
@ -6758,16 +6743,6 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
}
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
: SDNode(Opc, Order, dl, VTs, Ops),
MemoryVT(memvt), MMO(mmo) {
SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
MMO->isNonTemporal(), MMO->isInvariant());
assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
}
/// Profile - Gather unique data for the node.
///
void SDNode::Profile(FoldingSetNodeID &ID) const {