RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-12-03 17:02:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove "mask" operand from shufflevector.

Browse Source 
Instead, represent the mask as out-of-line data in the instruction. This
should be more efficient in the places that currently use
getShuffleVector(), and paves the way for further changes to add new
shuffles for scalable vectors.

This doesn't change the syntax in textual IR. And I don't currently plan
to change the bitcode encoding in this patch, although we'll probably
need to do something once we extend shufflevector for scalable types.

I expect that once this is finished, we can then replace the raw "mask"
with something more appropriate for scalable vectors.  Not sure exactly
what this looks like at the moment, but there are a few different ways
we could handle it.  Maybe we could try to describe specific shuffles.
Or maybe we could define it in terms of a function to convert a fixed-length
array into an appropriate scalable vector, using a "step", or something
like that.

Differential Revision: https://reviews.llvm.org/D72467
This commit is contained in:
Eli Friedman 2020-03-31 13:08:59 -07:00
parent dcd53dc437
commit db20f1e2c5
52 changed files with 672 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/llvm/ADT/ArrayRef.h
View File

@ -532,11 +532,21 @@ namespace llvm {
return LHS.equals(RHS);
}
template<typename T>
template <typename T>
inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
return ArrayRef<T>(LHS).equals(RHS);
}
template <typename T>
inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
template <typename T>
inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
/// @}
template <typename T> hash_code hash_value(ArrayRef<T> S) {

7
include/llvm/Analysis/ConstantFolding.h
View File

@ -119,10 +119,11 @@ Constant *ConstantFoldInsertElementInstruction(Constant *Val,
Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
/// Attempt to constant fold a shufflevector instruction with the
/// specified operands and indices. The constant result is returned if
/// successful; if not, null is returned.
/// specified operands and mask. See class ShuffleVectorInst for a description
/// of the mask representation. The constant result is returned if successful;
/// if not, null is returned.
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
Constant *Mask);
ArrayRef<int> Mask);
/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
/// produce if it is constant and determinable. If this is not determinable,

3
include/llvm/Analysis/InstructionSimplify.h
View File

@ -230,7 +230,8 @@ Value *SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
const SimplifyQuery &Q);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
/// See class ShuffleVectorInst for a description of the mask representation.
Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q);
//=== Helper functions for higher up the class hierarchy.

2
include/llvm/Analysis/TargetFolder.h
View File

@ -259,7 +259,7 @@ public:
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
Constant *Mask) const override {
ArrayRef<int> Mask) const override {
return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
}

2
include/llvm/IR/ConstantFolder.h
View File

@ -265,7 +265,7 @@ public:
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
Constant *Mask) const override {
ArrayRef<int> Mask) const override {
return ConstantExpr::getShuffleVector(V1, V2, Mask);
}

13
include/llvm/IR/Constants.h
View File

@ -1206,7 +1206,8 @@ public:
Type *OnlyIfReducedTy = nullptr);
static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
Type *OnlyIfReducedTy = nullptr);
static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
static Constant *getShuffleVector(Constant *V1, Constant *V2,
ArrayRef<int> Mask,
Type *OnlyIfReducedTy = nullptr);
static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
Type *OnlyIfReducedTy = nullptr);
@ -1225,6 +1226,16 @@ public:
/// expression and return the list of indices.
ArrayRef<unsigned> getIndices() const;
/// Assert that this is a shufflevector and return the mask. See class
/// ShuffleVectorInst for a description of the mask representation.
ArrayRef<int> getShuffleMask() const;
/// Assert that this is a shufflevector and return the mask.
///
/// TODO: This is a temporary hack until we update the bitcode format for
/// shufflevector.
Constant *getShuffleMaskForBitcode() const;
/// Return a string representation for an opcode.
const char *getOpcodeName() const;

24
include/llvm/IR/IRBuilder.h
View File

@ -2617,17 +2617,25 @@ public:
Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
const Twine &Name = "") {
if (auto *V1C = dyn_cast<Constant>(V1))
if (auto *V2C = dyn_cast<Constant>(V2))
if (auto *MC = dyn_cast<Constant>(Mask))
return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
SmallVector<int, 16> IntMask;
ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
return CreateShuffleVector(V1, V2, IntMask, Name);
}
Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
const Twine &Name = "") {
Value *Mask = ConstantDataVector::get(Context, IntMask);
return CreateShuffleVector(V1, V2, Mask, Name);
SmallVector<int, 16> IntMask;
IntMask.assign(Mask.begin(), Mask.end());
return CreateShuffleVector(V1, V2, IntMask, Name);
}
/// See class ShuffleVectorInst for a description of the mask representation.
Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &Name = "") {
if (auto *V1C = dyn_cast<Constant>(V1))
if (auto *V2C = dyn_cast<Constant>(V2))
return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}
Value *CreateExtractValue(Value *Agg,

2
include/llvm/IR/IRBuilderFolder.h
View File

@ -129,7 +129,7 @@ public:
virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt,
Constant *Idx) const = 0;
virtual Value *CreateShuffleVector(Constant *V1, Constant *V2,
Constant *Mask) const = 0;
ArrayRef<int> Mask) const = 0;
virtual Value *CreateExtractValue(Constant *Agg,
ArrayRef<unsigned> IdxList) const = 0;
virtual Value *CreateInsertValue(Constant *Agg, Constant *Val,

86
include/llvm/IR/Instructions.h
View File

@ -1988,10 +1988,22 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
// ShuffleVectorInst Class
//===----------------------------------------------------------------------===//
constexpr int UndefMaskElem = -1;
/// This instruction constructs a fixed permutation of two
/// input vectors.
///
/// For each element of the result vector, the shuffle mask selects an element
/// from one of the input vectors to copy to the result. Non-negative elements
/// in the mask represent an index into the concatenated pair of input vectors.
/// UndefMaskElem (-1) specifies that the result element is undefined.
///
/// For scalable vectors, all the elements of the mask must be 0 or -1. This
/// requirement may be relaxed in the future.
class ShuffleVectorInst : public Instruction {
SmallVector<int, 4> ShuffleMask;
Constant *ShuffleMaskForBitcode;
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
@ -2004,13 +2016,15 @@ public:
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &NameStr = "",
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
// allocate space for exactly three operands
void *operator new(size_t s) {
return User::operator new(s, 3);
}
void *operator new(size_t s) { return User::operator new(s, 2); }
/// Swap the first 2 operands and adjust the mask to preserve the semantics
/// Swap the operands and adjust the mask to preserve the semantics
/// of the instruction.
void commute();
@ -2018,6 +2032,8 @@ public:
/// formed with the specified operands.
static bool isValidOperands(const Value *V1, const Value *V2,
const Value *Mask);
static bool isValidOperands(const Value *V1, const Value *V2,
ArrayRef<int> Mask);
/// Overload to return most specific vector type.
///
@ -2028,36 +2044,33 @@ public:
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
Constant *getMask() const {
return cast<Constant>(getOperand(2));
}
/// Return the shuffle mask value for the specified element of the mask.
/// Return -1 if the element is undef.
static int getMaskValue(const Constant *Mask, unsigned Elt);
/// Return the shuffle mask value of this instruction for the given element
/// index. Return -1 if the element is undef.
int getMaskValue(unsigned Elt) const {
return getMaskValue(getMask(), Elt);
}
/// index. Return UndefMaskElem if the element is undef.
int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
/// Convert the input shuffle mask operand to a vector of integers. Undefined
/// elements of the mask are returned as -1.
/// elements of the mask are returned as UndefMaskElem.
static void getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result);
/// Return the mask for this instruction as a vector of integers. Undefined
/// elements of the mask are returned as -1.
/// elements of the mask are returned as UndefMaskElem.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
return getShuffleMask(getMask(), Result);
Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
SmallVector<int, 16> getShuffleMask() const {
SmallVector<int, 16> Mask;
getShuffleMask(Mask);
return Mask;
}
/// Return the mask for this instruction, for use in bitcode.
///
/// TODO: This is temporary until we decide a new bitcode encoding for
/// shufflevector.
Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
Type *ResultTy);
void setShuffleMask(ArrayRef<int> Mask);
ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
/// Return true if this shuffle returns a vector with a different number of
/// elements than its source vectors.
@ -2065,7 +2078,7 @@ public:
/// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
bool changesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts != NumMaskElts;
}
@ -2074,7 +2087,7 @@ public:
/// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
bool increasesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts < NumMaskElts;
}
@ -2095,7 +2108,7 @@ public:
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
return !changesLength() && isSingleSourceMask(getMask());
return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
@ -2116,7 +2129,7 @@ public:
/// from its input vectors.
/// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
bool isIdentity() const {
return !changesLength() && isIdentityMask(getShuffleMask());
return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
@ -2157,7 +2170,7 @@ public:
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
return !changesLength() && isSelectMask(getMask());
return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
@ -2177,7 +2190,7 @@ public:
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
return !changesLength() && isReverseMask(getMask());
return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
@ -2199,7 +2212,7 @@ public:
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
return !changesLength() && isZeroEltSplatMask(getMask());
return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
@ -2248,7 +2261,7 @@ public:
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
return !changesLength() && isTransposeMask(getMask());
return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is an extract subvector mask.
@ -2267,7 +2280,7 @@ public:
/// Return true if this shuffle mask is an extract subvector mask.
bool isExtractSubvectorMask(int &Index) const {
int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
}
/// Change values in a shuffle permute mask assuming the two vector operands
@ -2293,9 +2306,8 @@ public:
};
template <>
struct OperandTraits<ShuffleVectorInst> :
public FixedNumOperandTraits<ShuffleVectorInst, 3> {
};
struct OperandTraits<ShuffleVectorInst>
: public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)

2
include/llvm/IR/NoFolder.h
View File

@ -300,7 +300,7 @@ public:
}
Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
Constant *Mask) const override {
ArrayRef<int> Mask) const override {
return new ShuffleVectorInst(V1, V2, Mask);
}

71
include/llvm/IR/PatternMatch.h
View File

@ -50,6 +50,10 @@ template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
return const_cast<Pattern &>(P).match(V);
}
template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
return const_cast<Pattern &>(P).match(Mask);
}
template <typename SubPattern_t> struct OneUse_match {
SubPattern_t SubPattern;
@ -1350,12 +1354,69 @@ m_ExtractElement(const Val_t &Val, const Idx_t &Idx) {
return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
}
/// Matches ShuffleVectorInst.
/// Matches shuffle.
template <typename T0, typename T1, typename T2> struct Shuffle_match {
T0 Op1;
T1 Op2;
T2 Mask;
Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
: Op1(Op1), Op2(Op2), Mask(Mask) {}
template <typename OpTy> bool match(OpTy *V) {
if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
Mask.match(I->getShuffleMask());
}
return false;
}
};
struct m_Mask {
ArrayRef<int> &MaskRef;
m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
bool match(ArrayRef<int> Mask) {
MaskRef = Mask;
return true;
}
};
struct m_ZeroMask {
bool match(ArrayRef<int> Mask) {
return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
}
};
struct m_SpecificMask {
ArrayRef<int> &MaskRef;
m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
};
struct m_SplatOrUndefMask {
int &SplatIndex;
m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
bool match(ArrayRef<int> Mask) {
auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
if (First == Mask.end())
return false;
SplatIndex = *First;
return all_of(Mask,
[First](int Elem) { return Elem == *First || Elem == -1; });
}
};
/// Matches ShuffleVectorInst independently of mask value.
template <typename V1_t, typename V2_t>
inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
}
template <typename V1_t, typename V2_t, typename Mask_t>
inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
m);
inline Shuffle_match<V1_t, V2_t, Mask_t>
m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
}
/// Matches LoadInst.

3
lib/Analysis/ConstantFolding.cpp
View File

@ -1066,7 +1066,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
case Instruction::InsertElement:
return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
return ConstantExpr::getShuffleVector(
Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
}
}

47
lib/Analysis/InstructionSimplify.cpp
View File

@ -4439,36 +4439,31 @@ static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,
return RootVec;
}
static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
Type *RetTy, const SimplifyQuery &Q,
static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
ArrayRef<int> Mask, Type *RetTy,
const SimplifyQuery &Q,
unsigned MaxRecurse) {
if (isa<UndefValue>(Mask))
if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
return UndefValue::get(RetTy);
Type *InVecTy = Op0->getType();
ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
unsigned MaskNumElts = Mask.size();
ElementCount InVecEltCount = InVecTy->getVectorElementCount();
assert(MaskEltCount.Scalable == InVecEltCount.Scalable &&
"vscale mismatch between input vector and mask");
bool Scalable = MaskEltCount.Scalable;
bool Scalable = InVecEltCount.Scalable;
SmallVector<int, 32> Indices;
if (!Scalable) {
ShuffleVectorInst::getShuffleMask(Mask, Indices);
assert(MaskEltCount.Min == Indices.size() &&
"Size of Indices not same as number of mask elements?");
}
Indices.assign(Mask.begin(), Mask.end());
// Canonicalization: If mask does not select elements from an input vector,
// replace that input vector with undef.
if (!Scalable) {
// Canonicalization: If mask does not select elements from an input vector,
// replace that input vector with undef.
bool MaskSelects0 = false, MaskSelects1 = false;
for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
unsigned InVecNumElts = InVecEltCount.Min;
for (unsigned i = 0; i != MaskNumElts; ++i) {
if (Indices[i] == -1)
continue;
if ((unsigned)Indices[i] < InVecEltCount.Min)
if ((unsigned)Indices[i] < InVecNumElts)
MaskSelects0 = true;
else
MaskSelects1 = true;
@ -4514,8 +4509,8 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
// Shuffle mask undefs become undefined constant result elements.
SmallVector<Constant *, 16> VecC(MaskEltCount.Min, C);
for (unsigned i = 0; i != MaskEltCount.Min; ++i)
SmallVector<Constant *, 16> VecC(MaskNumElts, C);
for (unsigned i = 0; i != MaskNumElts; ++i)
if (Indices[i] == -1)
VecC[i] = UndefValue::get(C->getType());
return ConstantVector::get(VecC);
@ -4526,7 +4521,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
// value type is same as the input vectors' type.
if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
OpShuf->getMask()->getSplatValue())
is_splat(OpShuf->getShuffleMask()))
return Op0;
// All remaining transformation depend on the value of the mask, which is
@ -4545,7 +4540,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
// shuffle. This handles simple identity shuffles as well as chains of
// shuffles that may widen/narrow and/or move elements across lanes and back.
Value *RootVec = nullptr;
for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
for (unsigned i = 0; i != MaskNumElts; ++i) {
// Note that recursion is limited for each vector element, so if any element
// exceeds the limit, this will fail to simplify.
RootVec =
@ -4559,8 +4554,9 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
}
/// Given operands for a ShuffleVectorInst, fold the result or return null.
Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
Type *RetTy, const SimplifyQuery &Q) {
Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
ArrayRef<int> Mask, Type *RetTy,
const SimplifyQuery &Q) {
return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
}
@ -5523,8 +5519,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ,
}
case Instruction::ShuffleVector: {
auto *SVI = cast<ShuffleVectorInst>(I);
Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
SVI->getMask(), SVI->getType(), Q);
Result =
SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
SVI->getShuffleMask(), SVI->getType(), Q);
break;
}
case Instruction::PHI:

4
lib/Analysis/TargetTransformInfo.cpp
View File

@ -888,7 +888,7 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
SmallVector<int, 16> ActualMask = SI->getShuffleMask();
ArrayRef<int> ActualMask = SI->getShuffleMask();
return Mask == ActualMask;
}
@ -1153,7 +1153,7 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
// Fill the rest of the mask with -1 for undef.
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
ArrayRef<int> Mask = Shuffle->getShuffleMask();
if (ShuffleMask != Mask)
return RK_None;

6
lib/Analysis/ValueTracking.cpp
View File

@ -168,16 +168,16 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
APInt &DemandedLHS, APInt &DemandedRHS) {
// The length of scalable vectors is unknown at compile time, thus we
// cannot check their values
if (Shuf->getMask()->getType()->getVectorElementCount().Scalable)
if (Shuf->getType()->getVectorElementCount().Scalable)
return false;
int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
int NumMaskElts = Shuf->getType()->getVectorNumElements();
DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts);
if (DemandedElts.isNullValue())
return true;
// Simple case of a shuffle with zeroinitializer.
if (isa<ConstantAggregateZero>(Shuf->getMask())) {
if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) {
DemandedLHS.setBit(0);
return true;
}

8
lib/Analysis/VectorUtils.cpp
View File

@ -339,9 +339,9 @@ const llvm::Value *llvm::getSplatValue(const Value *V) {
// shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...>
Value *Splat;
if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
m_ZeroInt()),
m_Value(), m_ZeroInt())))
if (match(V, m_ShuffleVector(
m_InsertElement(m_Value(), m_Value(Splat), m_ZeroInt()),
m_Value(), m_ZeroMask())))
return Splat;
return nullptr;
@ -366,7 +366,7 @@ bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) {
if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
// FIXME: We can safely allow undefs here. If Index was specified, we will
// check that the mask elt is defined at the required index.
if (!Shuf->getMask()->getSplatValue())
if (!is_splat(Shuf->getShuffleMask()))
return false;
// Match any index.

5
lib/AsmParser/LLParser.cpp
View File

@ -3636,8 +3636,9 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
return Error(ID.Loc, "expected three operands to shufflevector");
if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2]))
return Error(ID.Loc, "invalid operands to shufflevector");
ID.ConstantVal =
ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]);
SmallVector<int, 16> Mask;
ShuffleVectorInst::getShuffleMask(cast<Constant>(Elts[2]), Mask);
ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask);
} else if (Opc == Instruction::ExtractElement) {
if (Elts.size() != 2)
return Error(ID.Loc, "expected two operands to extractelement");

52
lib/Bitcode/Reader/BitcodeReader.cpp
View File

@ -2337,6 +2337,15 @@ Error BitcodeReader::parseConstants() {
Type *CurFullTy = Type::getInt32Ty(Context);
unsigned NextCstNo = ValueList.size();
struct DelayedShufTy {
VectorType *OpTy;
VectorType *RTy;
Type *CurFullTy;
uint64_t Op0Idx;
uint64_t Op1Idx;
uint64_t Op2Idx;
};
std::vector<DelayedShufTy> DelayedShuffles;
while (true) {
Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
if (!MaybeEntry)
@ -2353,6 +2362,29 @@ Error BitcodeReader::parseConstants() {
// Once all the constants have been read, go through and resolve forward
// references.
//
// We have to treat shuffles specially because they don't have three
// operands anymore. We need to convert the shuffle mask into an array,
// and we can't convert a forward reference.
for (auto &DelayedShuffle : DelayedShuffles) {
VectorType *OpTy = DelayedShuffle.OpTy;
VectorType *RTy = DelayedShuffle.RTy;
uint64_t Op0Idx = DelayedShuffle.Op0Idx;
uint64_t Op1Idx = DelayedShuffle.Op1Idx;
uint64_t Op2Idx = DelayedShuffle.Op2Idx;
Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
Type *ShufTy =
VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount());
Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy);
if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2))
return error("Invalid shufflevector operands");
SmallVector<int, 16> Mask;
ShuffleVectorInst::getShuffleMask(Op2, Mask);
Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy);
++NextCstNo;
}
ValueList.resolveConstantForwardRefs();
return Error::success();
case BitstreamEntry::Record:
@ -2694,13 +2726,9 @@ Error BitcodeReader::parseConstants() {
VectorType *OpTy = dyn_cast<VectorType>(CurTy);
if (Record.size() < 3 || !OpTy)
return error("Invalid record");
Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
OpTy->getElementCount());
Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
break;
DelayedShuffles.push_back(
{OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]});
continue;
}
case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
VectorType *RTy = dyn_cast<VectorType>(CurTy);
@ -2708,13 +2736,9 @@ Error BitcodeReader::parseConstants() {
dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
if (Record.size() < 4 || !RTy || !OpTy)
return error("Invalid record");
Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
RTy->getElementCount());
Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
break;
DelayedShuffles.push_back(
{OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]});
continue;
}
case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred]
if (Record.size() < 4)

5
lib/Bitcode/Writer/BitcodeWriter.cpp
View File

@ -2515,7 +2515,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
}
Record.push_back(VE.getValueID(C->getOperand(0)));
Record.push_back(VE.getValueID(C->getOperand(1)));
Record.push_back(VE.getValueID(C->getOperand(2)));
Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
break;
case Instruction::ICmp:
case Instruction::FCmp:
@ -2702,7 +2702,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
pushValueAndType(I.getOperand(0), InstID, Vals);
pushValue(I.getOperand(1), InstID, Vals);
pushValue(I.getOperand(2), InstID, Vals);
pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
Vals);
break;
case Instruction::ICmp:
case Instruction::FCmp: {

43
lib/Bitcode/Writer/ValueEnumerator.cpp
View File

@ -88,11 +88,16 @@ static void orderValue(const Value *V, OrderMap &OM) {
if (OM.lookup(V).first)
return;
if (const Constant *C = dyn_cast<Constant>(V))
if (C->getNumOperands() && !isa<GlobalValue>(C))
if (const Constant *C = dyn_cast<Constant>(V)) {
if (C->getNumOperands() && !isa<GlobalValue>(C)) {
for (const Value *Op : C->operands())
if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op))
orderValue(Op, OM);
if (auto *CE = dyn_cast<ConstantExpr>(C))
if (CE->getOpcode() == Instruction::ShuffleVector)
orderValue(CE->getShuffleMaskForBitcode(), OM);
}
}
// Note: we cannot cache this lookup above, since inserting into the map
// changes the map's size, and thus affects the other IDs.
@ -155,11 +160,14 @@ static OrderMap orderModule(const Module &M) {
for (const Argument &A : F.args())
orderValue(&A, OM);
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
isa<InlineAsm>(*Op))
orderValue(Op, OM);
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
orderValue(SVI->getShuffleMaskForBitcode(), OM);
}
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
orderValue(&I, OM);
@ -250,11 +258,17 @@ static void predictValueUseListOrder(const Value *V, const Function *F,
predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack);
// Recursive descent into constants.
if (const Constant *C = dyn_cast<Constant>(V))
if (C->getNumOperands()) // Visit GlobalValues.
if (const Constant *C = dyn_cast<Constant>(V)) {
if (C->getNumOperands()) { // Visit GlobalValues.
for (const Value *Op : C->operands())
if (isa<Constant>(Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, F, OM, Stack);
if (auto *CE = dyn_cast<ConstantExpr>(C))
if (CE->getOpcode() == Instruction::ShuffleVector)
predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM,
Stack);
}
}
}
static UseListOrderStack predictUseListOrder(const Module &M) {
@ -279,10 +293,14 @@ static UseListOrderStack predictUseListOrder(const Module &M) {
for (const Argument &A : F.args())
predictValueUseListOrder(&A, &F, OM, Stack);
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, &F, OM, Stack);
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM,
Stack);
}
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
predictValueUseListOrder(&I, &F, OM, Stack);
@ -413,6 +431,8 @@ ValueEnumerator::ValueEnumerator(const Module &M,
EnumerateMetadata(&F, MD->getMetadata());
}
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
EnumerateType(I.getType());
if (const auto *Call = dyn_cast<CallBase>(&I))
EnumerateAttributes(Call->getAttributes());
@ -836,6 +856,9 @@ void ValueEnumerator::EnumerateValue(const Value *V) {
I != E; ++I)
if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
EnumerateValue(*I);
if (auto *CE = dyn_cast<ConstantExpr>(C))
if (CE->getOpcode() == Instruction::ShuffleVector)
EnumerateValue(CE->getShuffleMaskForBitcode());
// Finally, add the value. Doing this could make the ValueID reference be
// dangling, don't reuse it.
@ -913,6 +936,9 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) {
EnumerateOperandType(Op);
}
if (auto *CE = dyn_cast<ConstantExpr>(C))
if (CE->getOpcode() == Instruction::ShuffleVector)
EnumerateOperandType(CE->getShuffleMaskForBitcode());
}
void ValueEnumerator::EnumerateAttributes(AttributeList PAL) {
@ -958,11 +984,14 @@ void ValueEnumerator::incorporateFunction(const Function &F) {
// Add all function-level constants to the value table.
for (const BasicBlock &BB : F) {
for (const Instruction &I : BB)
for (const Instruction &I : BB) {
for (const Use &OI : I.operands()) {
if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI))
EnumerateValue(OI);
}
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
EnumerateValue(SVI->getShuffleMaskForBitcode());
}
BasicBlocks.push_back(&BB);
ValueMap[&BB] = BasicBlocks.size();
}

4
lib/CodeGen/CodeGenPrepare.cpp
View File

@ -6257,7 +6257,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
}
static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
SmallVector<int, 16> Mask(SVI->getShuffleMask());
ArrayRef<int> Mask(SVI->getShuffleMask());
int SplatElem = -1;
for (unsigned i = 0; i < Mask.size(); ++i) {
if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
@ -6307,7 +6307,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
assert(InsertPt != UserBB->end());
InsertedShuffle =
new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
SVI->getOperand(2), "", &*InsertPt);
SVI->getShuffleMask(), "", &*InsertPt);
InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
}

7
lib/CodeGen/GlobalISel/IRTranslator.cpp
View File

@ -1973,8 +1973,11 @@ bool IRTranslator::translateExtractElement(const User &U,
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
SmallVector<int, 8> Mask;
ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
ArrayRef<int> Mask;
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
Mask = SVI->getShuffleMask();
else
Mask = cast<ConstantExpr>(U).getShuffleMask();
ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
MIRBuilder
.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},

11
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View File

@ -3437,14 +3437,19 @@ void SelectionDAGBuilder::visitExtractElement(const User &I) {
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
Constant *MaskV = cast<Constant>(I.getOperand(2));
ArrayRef<int> Mask;
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
Mask = SVI->getShuffleMask();
else
Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();
unsigned SrcNumElts = SrcVT.getVectorNumElements();
if (MaskV->isNullValue() && VT.isScalableVector()) {
if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
VT.isScalableVector()) {
// Canonical splat form of first element of first input vector.
SDValue FirstElt =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
@ -3458,8 +3463,6 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
SmallVector<int, 8> Mask;
ShuffleVectorInst::getShuffleMask(MaskV, Mask);
unsigned MaskNumElts = Mask.size();
if (SrcNumElts == MaskNumElts) {

9
lib/ExecutionEngine/Interpreter/Execution.cpp
View File

@ -1868,7 +1868,6 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
GenericValue Dest;
// There is no need to check types of src1 and src2, because the compiled
@ -1878,7 +1877,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
Type *TyContained = Ty->getElementType();
unsigned src1Size = (unsigned)Src1.AggregateVal.size();
unsigned src2Size = (unsigned)Src2.AggregateVal.size();
unsigned src3Size = (unsigned)Src3.AggregateVal.size();
unsigned src3Size = I.getShuffleMask().size();
Dest.AggregateVal.resize(src3Size);
@ -1888,7 +1887,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::IntegerTyID:
for( unsigned i=0; i<src3Size; i++) {
unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal;
else if(j < src1Size + src2Size)
@ -1904,7 +1903,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::FloatTyID:
for( unsigned i=0; i<src3Size; i++) {
unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal;
else if(j < src1Size + src2Size)
@ -1915,7 +1914,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
break;
case Type::DoubleTyID:
for( unsigned i=0; i<src3Size; i++) {
unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal;
else if(j < src1Size + src2Size)

32
lib/IR/AsmWriter.cpp
View File

@ -462,6 +462,33 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) {
isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
}
static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
Out << ", <";
if (Ty->getVectorIsScalable())
Out << "vscale x ";
Out << Mask.size() << " x i32> ";
bool FirstElt = true;
if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
Out << "zeroinitializer";
} else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
Out << "undef";
} else {
Out << "<";
for (int Elt : Mask) {
if (FirstElt)
FirstElt = false;
else
Out << ", ";
Out << "i32 ";
if (Elt == UndefMaskElem)
Out << "undef";
else
Out << Elt;
}
Out << ">";
}
}
namespace {
class TypePrinting {
@ -1547,6 +1574,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
TypePrinter.print(CE->getType(), Out);
}
if (CE->getOpcode() == Instruction::ShuffleVector)
PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
Out << ')';
return;
}
@ -4093,6 +4123,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
RMWI->getSyncScopeID());
} else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
} else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
}
// Print Metadata info.

9
lib/IR/AutoUpgrade.cpp
View File

@ -2335,11 +2335,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
if (NumSrcElts == 2)
Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
{ 0, 1, 0, 1 });
Rep = Builder.CreateShuffleVector(
Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
else
Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
{ 0, 1, 2, 3, 0, 1, 2, 3 });
Rep =
Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();

23
lib/IR/ConstantFold.cpp
View File

@ -876,22 +876,22 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
return ConstantVector::get(Result);
}
Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
Constant *V2,
Constant *Mask) {
ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
ArrayRef<int> Mask) {
unsigned MaskNumElts = Mask.size();
ElementCount MaskEltCount = {MaskNumElts,
V1->getType()->getVectorIsScalable()};
Type *EltTy = V1->getType()->getVectorElementType();
// Undefined shuffle mask -> undefined value.
if (isa<UndefValue>(Mask))
return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
// Don't break the bitcode reader hack.
if (isa<ConstantExpr>(Mask)) return nullptr;
if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
}
// If the mask is all zeros this is a splat, no need to go through all
// elements.
if (isa<ConstantAggregateZero>(Mask) && !MaskEltCount.Scalable) {
if (all_of(Mask, [](int Elt) { return Elt == 0; }) &&
!MaskEltCount.Scalable) {
Type *Ty = IntegerType::get(V1->getContext(), 32);
Constant *Elt =
ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
@ -903,13 +903,12 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
if (ValTy->isScalable())
return nullptr;
unsigned MaskNumElts = MaskEltCount.Min;
unsigned SrcNumElts = V1->getType()->getVectorNumElements();
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
for (unsigned i = 0; i != MaskNumElts; ++i) {
int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
int Elt = Mask[i];
if (Elt == -1) {
Result.push_back(UndefValue::get(EltTy));
continue;

2
lib/IR/ConstantFold.h
View File

@ -38,7 +38,7 @@ template <typename T> class ArrayRef;
Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
Constant *Idx);
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
Constant *Mask);
ArrayRef<int> Mask);
Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
ArrayRef<unsigned> Idxs);
Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,

29
lib/IR/Constants.cpp
View File

@ -1230,8 +1230,7 @@ Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
Constant *UndefV = UndefValue::get(VTy);
V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
// Build shuffle mask to perform the splat.
Type *MaskTy = VectorType::get(I32Ty, EC);
Constant *Zeros = ConstantAggregateZero::get(MaskTy);
SmallVector<int, 8> Zeros(EC.Min, 0);
// Splat.
return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
}
@ -1298,6 +1297,14 @@ unsigned ConstantExpr::getPredicate() const {
return cast<CompareConstantExpr>(this)->predicate;
}
ArrayRef<int> ConstantExpr::getShuffleMask() const {
return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
}
Constant *ConstantExpr::getShuffleMaskForBitcode() const {
return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
}
Constant *
ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
assert(Op->getType() == getOperand(OpNo)->getType() &&
@ -1349,7 +1356,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
case Instruction::ExtractValue:
return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
case Instruction::ShuffleVector:
return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2],
return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
OnlyIfReducedTy);
case Instruction::GetElementPtr: {
auto *GEPO = cast<GEPOperator>(this);
@ -2152,7 +2159,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
if (InRangeIndex && *InRangeIndex < 63)
SubClassOptionalData |= (*InRangeIndex + 1) << 1;
const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
SubClassOptionalData, None, Ty);
SubClassOptionalData, None, None, Ty);
LLVMContextImpl *pImpl = C->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ReqTy, Key);
@ -2254,23 +2261,25 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt,
}
Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
Constant *Mask, Type *OnlyIfReducedTy) {
ArrayRef<int> Mask,
Type *OnlyIfReducedTy) {
assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector constant expr operands!");
if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
return FC; // Fold a few common cases.
ElementCount NElts = Mask->getType()->getVectorElementCount();
unsigned NElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
Type *ShufTy = VectorType::get(EltTy, NElts);
bool TypeIsScalable = V1->getType()->getVectorIsScalable();
Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
if (OnlyIfReducedTy == ShufTy)
return nullptr;
// Look up the constant in the table first to ensure uniqueness
Constant *ArgVec[] = { V1, V2, Mask };
const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
Constant *ArgVec[] = {V1, V2};
ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask);
LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
@ -3117,7 +3126,7 @@ Instruction *ConstantExpr::getAsInstruction() const {
case Instruction::ExtractValue:
return ExtractValueInst::Create(Ops[0], getIndices());
case Instruction::ShuffleVector:
return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
case Instruction::GetElementPtr: {
const auto *GO = cast<GEPOperator>(this);

86
lib/IR/ConstantsContext.h
View File

@ -26,6 +26,7 @@
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@ -146,21 +147,24 @@ public:
/// shufflevector constant exprs.
class ShuffleVectorConstantExpr : public ConstantExpr {
public:
ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
: ConstantExpr(VectorType::get(
cast<VectorType>(C1->getType())->getElementType(),
cast<VectorType>(C3->getType())->getElementCount()),
Instruction::ShuffleVector,
&Op<0>(), 3) {
ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask)
: ConstantExpr(
VectorType::get(cast<VectorType>(C1->getType())->getElementType(),
Mask.size(), C1->getType()->getVectorIsScalable()),
Instruction::ShuffleVector, &Op<0>(), 2) {
assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = C1;
Op<1>() = C2;
Op<2>() = C3;
ShuffleMask.assign(Mask.begin(), Mask.end());
ShuffleMaskForBitcode =
ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType());
}
// allocate space for exactly three operands
void *operator new(size_t s) {
return User::operator new(s, 3);
}
SmallVector<int, 4> ShuffleMask;
Constant *ShuffleMaskForBitcode;
void *operator new(size_t s) { return User::operator new(s, 2); }
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@ -319,7 +323,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value)
template <>
struct OperandTraits<ShuffleVectorConstantExpr>
: public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
: public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
template <>
@ -460,36 +464,58 @@ struct InlineAsmKeyType {
};
struct ConstantExprKeyType {
private:
uint8_t Opcode;
uint8_t SubclassOptionalData;
uint16_t SubclassData;
ArrayRef<Constant *> Ops;
ArrayRef<unsigned> Indexes;
ArrayRef<int> ShuffleMask;
Type *ExplicitTy;
static ArrayRef<int> getShuffleMaskIfValid(const ConstantExpr *CE) {
if (CE->getOpcode() == Instruction::ShuffleVector)
return CE->getShuffleMask();
return None;
}
static ArrayRef<unsigned> getIndicesIfValid(const ConstantExpr *CE) {
if (CE->hasIndices())
return CE->getIndices();
return None;
}
static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) {
if (auto *GEPCE = dyn_cast<GetElementPtrConstantExpr>(CE))
return GEPCE->getSourceElementType();
return nullptr;
}
public:
ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops,
unsigned short SubclassData = 0,
unsigned short SubclassOptionalData = 0,
ArrayRef<unsigned> Indexes = None,
ArrayRef<int> ShuffleMask = None,
Type *ExplicitTy = nullptr)
: Opcode(Opcode), SubclassOptionalData(SubclassOptionalData),
SubclassData(SubclassData), Ops(Ops), Indexes(Indexes),
ExplicitTy(ExplicitTy) {}
ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {}
ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands),
Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
ExplicitTy(nullptr) {}
Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
ExplicitTy(getSourceElementTypeIfValid(CE)) {}
ConstantExprKeyType(const ConstantExpr *CE,
SmallVectorImpl<Constant *> &Storage)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0),
Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
ExplicitTy(nullptr) {
Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
ExplicitTy(getSourceElementTypeIfValid(CE)) {
assert(Storage.empty() && "Expected empty storage");
for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
Storage.push_back(CE->getOperand(I));
@ -499,7 +525,8 @@ struct ConstantExprKeyType {
bool operator==(const ConstantExprKeyType &X) const {
return Opcode == X.Opcode && SubclassData == X.SubclassData &&
SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops &&
Indexes == X.Indexes;
Indexes == X.Indexes && ShuffleMask == X.ShuffleMask &&
ExplicitTy == X.ExplicitTy;
}
bool operator==(const ConstantExpr *CE) const {
@ -514,15 +541,21 @@ struct ConstantExprKeyType {
for (unsigned I = 0, E = Ops.size(); I != E; ++I)
if (Ops[I] != CE->getOperand(I))
return false;
if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()))
if (Indexes != getIndicesIfValid(CE))
return false;
if (ShuffleMask != getShuffleMaskIfValid(CE))
return false;
if (ExplicitTy != getSourceElementTypeIfValid(CE))
return false;
return true;
}
unsigned getHash() const {
return hash_combine(Opcode, SubclassOptionalData, SubclassData,
hash_combine_range(Ops.begin(), Ops.end()),
hash_combine_range(Indexes.begin(), Indexes.end()));
return hash_combine(
Opcode, SubclassOptionalData, SubclassData,
hash_combine_range(Ops.begin(), Ops.end()),
hash_combine_range(Indexes.begin(), Indexes.end()),
hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy);
}
using TypeClass = ConstantInfo<ConstantExpr>::TypeClass;
@ -546,17 +579,14 @@ struct ConstantExprKeyType {
case Instruction::InsertElement:
return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask);
case Instruction::InsertValue:
return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
case Instruction::ExtractValue:
return new ExtractValueConstantExpr(Ops[0], Indexes, Ty);
case Instruction::GetElementPtr:
return GetElementPtrConstantExpr::Create(
ExplicitTy ? ExplicitTy
: cast<PointerType>(Ops[0]->getType()->getScalarType())
->getElementType(),
Ops[0], Ops.slice(1), Ty, SubclassOptionalData);
return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1),
Ty, SubclassOptionalData);
case Instruction::ICmp:
return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData,
Ops[0], Ops[1]);

4
lib/IR/Core.cpp
View File

@ -1781,9 +1781,11 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant,
LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
LLVMValueRef VectorBConstant,
LLVMValueRef MaskConstant) {
SmallVector<int, 16> IntMask;
ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask);
return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant),
unwrap<Constant>(VectorBConstant),
unwrap<Constant>(MaskConstant)));
IntMask));
}
LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,

3
lib/IR/Instruction.cpp
View File

@ -443,6 +443,9 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2,
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
return SVI->getShuffleMask() ==
cast<ShuffleVectorInst>(I2)->getShuffleMask();
return true;
}

153
lib/IR/Instructions.cpp
View File

@ -1843,57 +1843,109 @@ bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt,
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &Name,
Instruction *InsertBefore)
: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
cast<VectorType>(Mask->getType())->getElementCount()),
ShuffleVector,
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertBefore) {
: Instruction(
VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
cast<VectorType>(Mask->getType())->getElementCount()),
ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
Op<2>() = Mask;
SmallVector<int, 16> MaskArr;
getShuffleMask(cast<Constant>(Mask), MaskArr);
setShuffleMask(MaskArr);
setName(Name);
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &Name,
BasicBlock *InsertAtEnd)
: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
cast<VectorType>(Mask->getType())->getElementCount()),
ShuffleVector,
OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this),
InsertAtEnd) {
const Twine &Name, BasicBlock *InsertAtEnd)
: Instruction(
VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
cast<VectorType>(Mask->getType())->getElementCount()),
ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
Op<2>() = Mask;
SmallVector<int, 16> MaskArr;
getShuffleMask(cast<Constant>(Mask), MaskArr);
setShuffleMask(MaskArr);
setName(Name);
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &Name,
Instruction *InsertBefore)
: Instruction(
VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
Mask.size(), V1->getType()->getVectorIsScalable()),
ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
setShuffleMask(Mask);
setName(Name);
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &Name, BasicBlock *InsertAtEnd)
: Instruction(
VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
Mask.size(), V1->getType()->getVectorIsScalable()),
ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
setShuffleMask(Mask);
setName(Name);
}
void ShuffleVectorInst::commute() {
int NumOpElts = Op<0>()->getType()->getVectorNumElements();
int NumMaskElts = getMask()->getType()->getVectorNumElements();
SmallVector<Constant*, 16> NewMask(NumMaskElts);
Type *Int32Ty = Type::getInt32Ty(getContext());
int NumMaskElts = ShuffleMask.size();
SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
if (MaskElt == -1) {
NewMask[i] = UndefValue::get(Int32Ty);
if (MaskElt == UndefMaskElem) {
NewMask[i] = UndefMaskElem;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
NewMask[i] = MaskElt;
}
Op<2>() = ConstantVector::get(NewMask);
setShuffleMask(NewMask);
Op<0>().swap(Op<1>());
}
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
ArrayRef<int> Mask) {
// V1 and V2 must be vectors of the same type.
if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
return false;
// Make sure the mask elements make sense.
int V1Size = cast<VectorType>(V1->getType())->getNumElements();
for (int Elem : Mask)
if (Elem != UndefMaskElem && Elem >= V1Size * 2)
return false;
if (V1->getType()->getVectorIsScalable())
if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask))
return false;
return true;
}
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
const Value *Mask) {
// V1 and V2 must be vectors of the same type.
@ -1902,7 +1954,8 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
// Mask must be vector of i32.
auto *MaskTy = dyn_cast<VectorType>(Mask->getType());
if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32))
if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) ||
MaskTy->isScalable() != V1->getType()->getVectorIsScalable())
return false;
// Check to see if Mask is valid.
@ -1930,34 +1983,12 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
return true;
}
// The bitcode reader can create a place holder for a forward reference
// used as the shuffle mask. When this occurs, the shuffle mask will
// fall into this case and fail. To avoid this error, do this bit of
// ugliness to allow such a mask pass.
if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
if (CE->getOpcode() == Instruction::UserOp1)
return true;
return false;
}
int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) {
assert(i < Mask->getType()->getVectorNumElements() && "Index out of range");
assert(!Mask->getType()->getVectorElementCount().Scalable &&
"Length of scalable vectors unknown at compile time");
if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask))
return CDS->getElementAsInteger(i);
Constant *C = Mask->getAggregateElement(i);
if (isa<UndefValue>(C))
return -1;
return cast<ConstantInt>(C)->getZExtValue();
}
void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result) {
assert(!Mask->getType()->getVectorElementCount().Scalable &&
"Length of scalable vectors unknown at compile time");
unsigned NumElts = Mask->getType()->getVectorNumElements();
unsigned NumElts = Mask->getType()->getVectorElementCount().Min;
if (isa<ConstantAggregateZero>(Mask)) {
Result.resize(NumElts, 0);
return;
@ -1975,6 +2006,30 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
}
}
void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
ShuffleMask.assign(Mask.begin(), Mask.end());
ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType());
}
Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
Type *ResultTy) {
Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext());
if (ResultTy->getVectorIsScalable()) {
assert(is_splat(Mask) && "Unexpected shuffle");
Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true);
if (Mask[0] == 0)
return Constant::getNullValue(VecTy);
return UndefValue::get(VecTy);
}
SmallVector<Constant *, 16> MaskConst;
for (int Elem : Mask) {
if (Elem == UndefMaskElem)
MaskConst.push_back(UndefValue::get(Int32Ty));
else
MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
}
return ConstantVector::get(MaskConst);
}
static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
assert(!Mask.empty() && "Shuffle mask must contain elements");
bool UsesLHS = false;
@ -2124,7 +2179,7 @@ bool ShuffleVectorInst::isIdentityWithPadding() const {
return false;
// The first part of the mask must choose elements from exactly 1 source op.
SmallVector<int, 16> Mask = getShuffleMask();
ArrayRef<int> Mask = getShuffleMask();
if (!isIdentityMaskImpl(Mask, NumOpElts))
return false;
@ -4298,7 +4353,7 @@ InsertElementInst *InsertElementInst::cloneImpl() const {
}
ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
}
PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }

2
lib/IR/Verifier.cpp
View File

@ -3310,7 +3310,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) {
void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
SV.getOperand(2)),
SV.getShuffleMask()),
"Invalid shufflevector operands!", &SV);
visitInstruction(SV);
}

6
lib/Target/AArch64/AArch64ISelLowering.cpp
View File

@ -9207,10 +9207,10 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
};
Constant *M1, *M2;
ArrayRef<int> M1, M2;
Value *S1Op1, *S2Op1;
if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
!match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
!match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
return false;
// Check that the operands are half as wide as the result and we extract

2
lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
View File

@ -210,7 +210,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
Arg.replaceAllUsesWith(NewVal);
} else if (IsV3) {
Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
{0, 1, 2},
ArrayRef<int>{0, 1, 2},
Arg.getName() + ".load");
Arg.replaceAllUsesWith(Shuf);
} else {

2
lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
View File

@ -409,7 +409,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
DL->getTypeSizeInBits(Val->getType())) {
assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
{ 0, 1, 2 });
ArrayRef<int>{0, 1, 2});
}
Val = B.CreateBitCast(Val, EffectiveEltTy);

2
lib/Target/ARM/ARMISelLowering.cpp
View File

@ -15547,7 +15547,7 @@ bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
return false;
if (!match(I->getOperand(Op),
m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
m_Undef(), m_Zero()))) {
m_Undef(), m_ZeroMask()))) {
return false;
}
Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));

4
lib/Target/ARM/MVETailPredication.cpp
View File

@ -286,7 +286,7 @@ bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
Instruction *Insert = nullptr;
// The shuffle which broadcasts the index iv into a vector.
if (!match(BroadcastSplat,
m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// The insert element which initialises a vector with the index iv.
@ -409,7 +409,7 @@ static bool MatchElemCountLoopSetup(Loop *L, Instruction *Shuffle,
Instruction *Insert = nullptr;
if (!match(Shuffle,
m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// Insert the limit into a vector.

2
lib/Target/X86/X86PartialReduction.cpp
View File

@ -384,7 +384,7 @@ bool X86PartialReduction::trySADReplacement(Value *Op, BinaryOperator *Add) {
NumElts = OpTy->getVectorNumElements();
if (NumElts == 2) {
// Extract down to 2 elements.
Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], {0, 1});
Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{0, 1});
} else if (NumElts >= 8) {
SmallVector<uint32_t, 32> ConcatMask(NumElts);
unsigned SubElts = Ops[0]->getType()->getVectorNumElements();

6
lib/Transforms/InstCombine/InstCombineCasts.cpp
View File

@ -642,12 +642,12 @@ static Instruction *shrinkSplatShuffle(TruncInst &Trunc,
InstCombiner::BuilderTy &Builder) {
auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
Shuf->getMask()->getSplatValue() &&
is_splat(Shuf->getShuffleMask()) &&
Shuf->getType() == Shuf->getOperand(0)->getType()) {
// trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
Constant *NarrowUndef = UndefValue::get(Trunc.getType());
Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
}
return nullptr;
@ -2553,7 +2553,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
// Return a new shuffle vector. Use the same element ID's, as we
// know the vector types match #elts.
return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
}
}

24
lib/Transforms/InstCombine/InstCombineCompares.cpp
View File

@ -2814,11 +2814,10 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
return nullptr;
Value *Vec;
Constant *Mask;
if (match(BCSrcOp,
m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
ArrayRef<int> Mask;
if (match(BCSrcOp, m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
// Check whether every element of Mask is the same constant
if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
if (is_splat(Mask)) {
auto *VecTy = cast<VectorType>(BCSrcOp->getType());
auto *EltTy = cast<IntegerType>(VecTy->getElementType());
if (C->isSplat(EltTy->getBitWidth())) {
@ -2827,6 +2826,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
// then:
// => %E = extractelement <N x iK> %vec, i32 Elem
// icmp <pred> iK %SplatVal, <pattern>
Value *Elem = Builder.getInt32(Mask[0]);
Value *Extract = Builder.CreateExtractElement(Vec, Elem);
Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
return new ICmpInst(Pred, Extract, NewC);
@ -5380,15 +5380,15 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
bool IsFP = isa<FCmpInst>(Cmp);
Value *V1, *V2;
Constant *M;
if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))))
ArrayRef<int> M;
if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))))
return nullptr;
// If both arguments of the cmp are shuffles that use the same mask and
// shuffle within a single vector, move the shuffle after the cmp:
// cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
Type *V1Ty = V1->getType();
if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2)
: Builder.CreateICmp(Pred, V1, V2);
@ -5405,16 +5405,16 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
// Length-changing splats are ok, so adjust the constants as needed:
// cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true);
if (ScalarC && ScalarM) {
int MaskSplatIndex;
if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
// We allow undefs in matching, but this transform removes those for safety.
// Demanded elements analysis should be able to recover some/all of that.
C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
ScalarM);
SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
: Builder.CreateICmp(Pred, V1, C);
return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);
return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()),
NewM);
}
return nullptr;

11
lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
View File

@ -1389,7 +1389,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
Shuffle->getOperand(0)->getType()->getVectorNumElements();
// Handle trivial case of a splat. Only check the first element of LHS
// operand.
if (isa<ConstantAggregateZero>(Shuffle->getMask()) &&
if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) &&
DemandedElts.isAllOnesValue()) {
if (!isa<UndefValue>(I->getOperand(1))) {
I->setOperand(1, UndefValue::get(I->getOperand(1)->getType()));
@ -1518,15 +1518,14 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
}
if (NewUndefElts) {
// Add additional discovered undefs.
SmallVector<Constant*, 16> Elts;
SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
Elts.push_back(UndefMaskElem);
else
Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
Shuffle->getMaskValue(i)));
Elts.push_back(Shuffle->getMaskValue(i));
}
I->setOperand(2, ConstantVector::get(Elts));
Shuffle->setShuffleMask(Elts);
MadeChange = true;
}
break;

124
lib/Transforms/InstCombine/InstCombineVectorOps.cpp
View File

@ -730,7 +730,7 @@ Instruction *InstCombiner::visitInsertValueInst(InsertValueInst &I) {
}
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
int MaskSize = Shuf.getShuffleMask().size();
int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
// A vector select does not change the size of the operands.
@ -842,13 +842,10 @@ static Instruction *foldInsEltIntoSplat(InsertElementInst &InsElt) {
// inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
// --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
Constant *Zero = ConstantInt::getNullValue(I32Ty);
SmallVector<int, 16> NewMask(NumMaskElts);
for (unsigned i = 0; i != NumMaskElts; ++i)
NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
}
@ -878,26 +875,23 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
// For example:
// inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
Constant *OldMask = Shuf->getMask();
SmallVector<int, 16> NewMask(NumMaskElts);
ArrayRef<int> OldMask = Shuf->getShuffleMask();
for (unsigned i = 0; i != NumMaskElts; ++i) {
if (i != IdxC) {
// All mask elements besides the inserted element remain the same.
NewMaskVec[i] = OldMask->getAggregateElement(i);
} else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
NewMask[i] = OldMask[i];
} else if (OldMask[i] == (int)IdxC) {
// If the mask element was already set, there's nothing to do
// (demanded elements analysis may unset it later).
return nullptr;
} else {
assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
assert(OldMask[i] == UndefMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
NewMaskVec[i] = NewMaskEltC;
NewMask[i] = IdxC;
}
}
Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
}
@ -965,27 +959,25 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) {
// mask vector with the insertelt index plus the length of the vector
// (because the constant vector operand of a shuffle is always the 2nd
// operand).
Constant *Mask = Shuf->getMask();
unsigned NumElts = Mask->getType()->getVectorNumElements();
ArrayRef<int> Mask = Shuf->getShuffleMask();
unsigned NumElts = Mask.size();
SmallVector<Constant *, 16> NewShufElts(NumElts);
SmallVector<Constant *, 16> NewMaskElts(NumElts);
SmallVector<int, 16> NewMaskElts(NumElts);
for (unsigned I = 0; I != NumElts; ++I) {
if (I == InsEltIndex) {
NewShufElts[I] = InsEltScalar;
Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
NewMaskElts[I] = InsEltIndex + NumElts;
} else {
// Copy over the existing values.
NewShufElts[I] = ShufConstVec->getAggregateElement(I);
NewMaskElts[I] = Mask->getAggregateElement(I);
NewMaskElts[I] = Mask[I];
}
}
// Create new operands for a shuffle that includes the constant of the
// original insertelt. The old shuffle will be dead now.
return new ShuffleVectorInst(Shuf->getOperand(0),
ConstantVector::get(NewShufElts),
ConstantVector::get(NewMaskElts));
ConstantVector::get(NewShufElts), NewMaskElts);
} else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
// Transform sequences of insertelements ops with constant data/indexes into
// a single shuffle op.
@ -1305,17 +1297,9 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
if (isa<ConstantAggregateZero>(V))
return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size()));
if (Constant *C = dyn_cast<Constant>(V)) {
SmallVector<Constant *, 16> MaskValues;
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
MaskValues.push_back(UndefValue::get(I32Ty));
else
MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i]));
}
if (Constant *C = dyn_cast<Constant>(V))
return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()),
ConstantVector::get(MaskValues));
}
Mask);
Instruction *I = cast<Instruction>(V);
switch (I->getOpcode()) {
@ -1404,7 +1388,7 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
// Shuffles to: |EE|FF|GG|HH|
// +--+--+--+--+
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
SmallVector<int, 16> &Mask) {
ArrayRef<int> Mask) {
unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
unsigned MaskElems = Mask.size();
unsigned BegIdx = Mask.front();
@ -1487,12 +1471,12 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
// Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
// The existing binop constant vector remains in the same operand position.
Constant *Mask = Shuf.getMask();
ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
Mask->containsUndefElement() &&
is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
@ -1506,7 +1490,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@ -1518,14 +1502,14 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
InstCombiner::BuilderTy &Builder) {
Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
Constant *Mask = Shuf.getMask();
ArrayRef<int> Mask = Shuf.getShuffleMask();
Value *X;
uint64_t IndexC;
// Match a shuffle that is a splat to a non-zero element.
if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
m_ConstantInt(IndexC)))) ||
!match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
!match(Op1, m_Undef()) || match(Mask, m_ZeroMask()) || IndexC == 0)
return nullptr;
// Insert into element 0 of an undef vector.
@ -1538,12 +1522,12 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
// shuf (inselt undef, X, 2), undef, <2,2,undef>
// --> shuf (inselt undef, X, 0), undef, <0,0,undef>
unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
if (isa<UndefValue>(Mask->getAggregateElement(i)))
NewMask[i] = Mask->getAggregateElement(i);
if (Mask[i] == UndefMaskElem)
NewMask[i] = Mask[i];
return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
}
/// Try to fold shuffles that are the equivalent of a vector select.
@ -1612,14 +1596,14 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
BinaryOperator::BinaryOps BOpc = Opc0;
// Select the constant elements needed for the single binop.
Constant *Mask = Shuf.getMask();
ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
// We are moving a binop after a shuffle. When a shuffle has an undefined
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
Mask->containsUndefElement() &&
is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
@ -1668,7 +1652,7 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
NewBO->andIRFlags(B1);
if (DropNSW)
NewBO->setHasNoSignedWrap(false);
if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@ -1694,8 +1678,7 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
// and have the same number of elements as this shuffle.
unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
Value *NarrowCond;
if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
m_Constant()))) ||
if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
!cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
return nullptr;
@ -1703,8 +1686,8 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
// shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
// sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
Value *Undef = UndefValue::get(X->getType());
Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
@ -1715,8 +1698,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
return nullptr;
Value *X, *Y;
Constant *Mask;
if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
ArrayRef<int> Mask;
if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Mask(Mask))))
return nullptr;
// Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
@ -1736,16 +1719,16 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
// shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
// shuf X, Y, <C0, undef, C2, undef>
unsigned NumElts = Shuf.getType()->getVectorNumElements();
SmallVector<Constant *, 16> NewMask(NumElts);
assert(NumElts < Mask->getType()->getVectorNumElements() &&
SmallVector<int, 16> NewMask(NumElts);
assert(NumElts < Mask.size() &&
"Identity with extract must have less elements than its inputs");
for (unsigned i = 0; i != NumElts; ++i) {
Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
Constant *MaskElt = Mask->getAggregateElement(i);
NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
int ExtractMaskElt = Shuf.getMaskValue(i);
int MaskElt = Mask[i];
NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
}
return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
return new ShuffleVectorInst(X, Y, NewMask);
}
/// Try to replace a shuffle with an insertelement or try to replace a shuffle
@ -1753,7 +1736,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
InstCombiner &IC) {
Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
SmallVector<int, 16> Mask = Shuf.getShuffleMask();
SmallVector<int, 16> Mask;
Shuf.getShuffleMask(Mask);
// The shuffle must not change vector sizes.
// TODO: This restriction could be removed if the insert has only one use
@ -1872,7 +1856,7 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
SmallVector<int, 16> Mask = Shuf.getShuffleMask();
ArrayRef<int> Mask = Shuf.getShuffleMask();
SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
@ -1905,28 +1889,28 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
Value *LHS = SVI.getOperand(0);
Value *RHS = SVI.getOperand(1);
if (auto *V = SimplifyShuffleVectorInst(
LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
if (auto *V =
SimplifyShuffleVectorInst(LHS, RHS, SVI.getShuffleMask(),
SVI.getType(), SQ.getWithInstruction(&SVI)))
return replaceInstUsesWith(SVI, V);
// shuffle x, x, mask --> shuffle x, undef, mask'
unsigned VWidth = SVI.getType()->getVectorNumElements();
unsigned LHSWidth = LHS->getType()->getVectorNumElements();
SmallVector<int, 16> Mask = SVI.getShuffleMask();
ArrayRef<int> Mask = SVI.getShuffleMask();
Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
if (LHS == RHS) {
assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
// Remap any references to RHS to use LHS.
SmallVector<Constant*, 16> Elts;
SmallVector<int, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) {
// Propagate undef elements or force mask to LHS.
if (Mask[i] < 0)
Elts.push_back(UndefValue::get(Int32Ty));
Elts.push_back(UndefMaskElem);
else
Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
Elts.push_back(Mask[i] % LHSWidth);
}
return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()),
ConstantVector::get(Elts));
return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), Elts);
}
// shuffle undef, x, mask --> shuffle x, undef, mask'
@ -2151,8 +2135,8 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
if (newLHS == LHS && newRHS == RHS)
return MadeChange ? &SVI : nullptr;
SmallVector<int, 16> LHSMask;
SmallVector<int, 16> RHSMask;
ArrayRef<int> LHSMask;
ArrayRef<int> RHSMask;
if (newLHS != LHS)
LHSMask = LHSShuffle->getShuffleMask();
if (RHSShuffle && newRHS != RHS)

47
lib/Transforms/InstCombine/InstructionCombining.cpp
View File

@ -1534,9 +1534,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// narrow binop on each pair of the source operands followed by concatenation
// of the results.
Value *L0, *L1, *R0, *R1;
Constant *Mask;
if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
ArrayRef<int> Mask;
if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
match(RHS,
m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
LHS->hasOneUse() && RHS->hasOneUse() &&
cast<ShuffleVectorInst>(LHS)->isConcat() &&
cast<ShuffleVectorInst>(RHS)->isConcat()) {
@ -1560,7 +1561,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
Value *XY = Builder.CreateBinOp(Opcode, X, Y);
if (auto *BO = dyn_cast<BinaryOperator>(XY))
BO->copyIRFlags(&Inst);
@ -1570,8 +1571,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// If both arguments of the binary operation are shuffles that use the same
// mask and shuffle within a single vector, move the shuffle after the binop.
Value *V1, *V2;
if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
match(RHS,
m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
// Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
@ -1581,17 +1583,19 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// If both arguments of a commutative binop are select-shuffles that use the
// same mask with commuted operands, the shuffles are unnecessary.
if (Inst.isCommutative() &&
match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
m_Specific(Mask)))) {
m_SpecificMask(Mask)))) {
auto *LShuf = cast<ShuffleVectorInst>(LHS);
auto *RShuf = cast<ShuffleVectorInst>(RHS);
// TODO: Allow shuffles that contain undefs in the mask?
// That is legal, but it reduces undef knowledge.
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
if (LShuf->isSelect() &&
!is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
RShuf->isSelect() &&
!is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
@ -1608,9 +1612,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// other binops, so they can be folded. It may also enable demanded elements
// transforms.
Constant *C;
if (match(&Inst, m_c_BinOp(
m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
m_Constant(C))) &&
if (match(&Inst, m_c_BinOp(m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(),
m_Mask(Mask))),
m_Constant(C))) &&
V1->getType()->getVectorNumElements() <= NumElts) {
assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
"Shuffle should not change scalar type");
@ -1621,8 +1625,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// reorder is not possible. A 1-to-1 mapping is not required. Example:
// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
bool ConstOp1 = isa<Constant>(RHS);
SmallVector<int, 16> ShMask;
ShuffleVectorInst::getShuffleMask(Mask, ShMask);
ArrayRef<int> ShMask = Mask;
unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
@ -1687,12 +1690,12 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
std::swap(LHS, RHS);
Value *X;
Constant *MaskC;
const APInt *SplatIndex;
ArrayRef<int> MaskC;
int SplatIndex;
BinaryOperator *BO;
if (!match(LHS, m_OneUse(m_ShuffleVector(m_Value(X), m_Undef(),
m_Constant(MaskC)))) ||
!match(MaskC, m_APIntAllowUndef(SplatIndex)) ||
m_Mask(MaskC)))) ||
!match(MaskC, m_SplatOrUndefMask(SplatIndex)) ||
X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) ||
BO->getOpcode() != Opcode)
return nullptr;
@ -1701,10 +1704,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// moving 'Y' before the splat shuffle, we are implicitly assuming
// that it is not undef/poison at the splat index.
Value *Y, *OtherOp;
if (isSplatValue(BO->getOperand(0), SplatIndex->getZExtValue())) {
if (isSplatValue(BO->getOperand(0), SplatIndex)) {
Y = BO->getOperand(0);
OtherOp = BO->getOperand(1);
} else if (isSplatValue(BO->getOperand(1), SplatIndex->getZExtValue())) {
} else if (isSplatValue(BO->getOperand(1), SplatIndex)) {
Y = BO->getOperand(1);
OtherOp = BO->getOperand(0);
} else {
@ -1716,7 +1719,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
// bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
UndefValue *Undef = UndefValue::get(Inst.getType());
Constant *NewMask = ConstantInt::get(MaskC->getType(), *SplatIndex);
SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
Value *NewSplat = Builder.CreateShuffleVector(NewBO, Undef, NewMask);
Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);

3
lib/Transforms/Instrumentation/MemorySanitizer.cpp
View File

@ -1867,10 +1867,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
void visitShuffleVectorInst(ShuffleVectorInst &I) {
insertShadowCheck(I.getOperand(2), &I);
IRBuilder<> IRB(&I);
setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
I.getOperand(2), "_msprop"));
I.getShuffleMask(), "_msprop"));
setOriginForNaryOp(I);
}

14
lib/Transforms/Scalar/GVN.cpp
View File

@ -289,7 +289,7 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
e.commutative = true;
}
if (CmpInst *C = dyn_cast<CmpInst>(I)) {
if (auto *C = dyn_cast<CmpInst>(I)) {
// Sort the operand value numbers so x<y and y>x get the same value number.
CmpInst::Predicate Predicate = C->getPredicate();
if (e.varargs[0] > e.varargs[1]) {
@ -298,10 +298,11 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
}
e.opcode = (C->getOpcode() << 8) | Predicate;
e.commutative = true;
} else if (InsertValueInst *E = dyn_cast<InsertValueInst>(I)) {
for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
II != IE; ++II)
e.varargs.push_back(*II);
} else if (auto *E = dyn_cast<InsertValueInst>(I)) {
e.varargs.append(E->idx_begin(), E->idx_end());
} else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
}
return e;
@ -1732,7 +1733,8 @@ uint32_t GVN::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
// instead of value numbers. Those index numbers should not be
// translated.
if ((i > 1 && Exp.opcode == Instruction::InsertValue) ||
(i > 0 && Exp.opcode == Instruction::ExtractValue))
(i > 0 && Exp.opcode == Instruction::ExtractValue) ||
(i > 1 && Exp.opcode == Instruction::ShuffleVector))
continue;
Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn);
}

12
lib/Transforms/Scalar/GVNSink.cpp
View File

@ -350,6 +350,7 @@ using ModelledPHISet = DenseSet<ModelledPHI, DenseMapInfo<ModelledPHI>>;
class InstructionUseExpr : public GVNExpression::BasicExpression {
unsigned MemoryUseOrder = -1;
bool Volatile = false;
std::vector<int> ShuffleMask;
public:
InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
@ -366,15 +367,18 @@ public:
void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; }
void setVolatile(bool V) { Volatile = V; }
void setShuffleMask(ArrayRef<int> Mask) {
ShuffleMask.assign(Mask.begin(), Mask.end());
}
hash_code getHashValue() const override {
return hash_combine(GVNExpression::BasicExpression::getHashValue(),
MemoryUseOrder, Volatile);
MemoryUseOrder, Volatile, ArrayRef<int>(ShuffleMask));
}
template <typename Function> hash_code getHashValue(Function MapFn) {
hash_code H =
hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile,
ArrayRef<int>(ShuffleMask));
for (auto *V : operands())
H = hash_combine(H, MapFn(V));
return H;
@ -402,6 +406,8 @@ class ValueTable {
CmpInst::Predicate Predicate = C->getPredicate();
E->setOpcode((C->getOpcode() << 8) | Predicate);
}
if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
E->setShuffleMask(SVI->getShuffleMask());
return E;
}

4
lib/Transforms/Scalar/NewGVN.cpp
View File

@ -2030,10 +2030,12 @@ NewGVN::performSymbolicEvaluation(Value *V,
case Instruction::Select:
case Instruction::ExtractElement:
case Instruction::InsertElement:
case Instruction::ShuffleVector:
case Instruction::GetElementPtr:
E = createExpression(I);
break;
case Instruction::ShuffleVector:
// FIXME: Add support for shufflevector to createExpression.
return nullptr;
default:
return nullptr;
}

2
lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
View File

@ -1009,7 +1009,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
auto *SV = cast<ShuffleVectorInst>(I);
UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
std::string Name = suffixed_name_or(I, ".base", "base_sv");
return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
Name, SV);
}
};

2
lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -3405,7 +3405,7 @@ void InnerLoopVectorizer::truncateToMinimalBitwidths() {
auto *O1 = B.CreateZExtOrTrunc(
SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
} else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
// Don't do anything with the operands, just extend the result.
continue;

17
unittests/IR/PatternMatch.cpp
View File

@ -991,31 +991,32 @@ TEST_F(PatternMatchTest, VectorOps) {
EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
// Test matching shufflevector
EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
ArrayRef<int> Mask;
EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
EXPECT_TRUE(
match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
EXPECT_TRUE(A == VI3);
EXPECT_TRUE(B == VI4);
EXPECT_TRUE(C == IdxVec);
A = B = C = nullptr; // reset
// Test matching the vector splat pattern
EXPECT_TRUE(match(
SI1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
m_Undef(), m_Zero())));
m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
m_Undef(), m_Zero())));
m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
m_Undef(), m_Zero())));
m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
m_Undef(), m_Zero())));
m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
m_Undef(), m_Zero())));
m_Undef(), m_ZeroMask())));
EXPECT_TRUE(A == Val);
}