From db20f1e2c5a5ef5120a1c8b497ffc3966c029364 Mon Sep 17 00:00:00 2001 From: Eli Friedman Date: Tue, 31 Mar 2020 13:08:59 -0700 Subject: [PATCH] Remove "mask" operand from shufflevector. Instead, represent the mask as out-of-line data in the instruction. This should be more efficient in the places that currently use getShuffleVector(), and paves the way for further changes to add new shuffles for scalable vectors. This doesn't change the syntax in textual IR. And I don't currently plan to change the bitcode encoding in this patch, although we'll probably need to do something once we extend shufflevector for scalable types. I expect that once this is finished, we can then replace the raw "mask" with something more appropriate for scalable vectors. Not sure exactly what this looks like at the moment, but there are a few different ways we could handle it. Maybe we could try to describe specific shuffles. Or maybe we could define it in terms of a function to convert a fixed-length array into an appropriate scalable vector, using a "step", or something like that. Differential Revision: https://reviews.llvm.org/D72467 --- include/llvm/ADT/ArrayRef.h | 12 +- include/llvm/Analysis/ConstantFolding.h | 7 +- include/llvm/Analysis/InstructionSimplify.h | 3 +- include/llvm/Analysis/TargetFolder.h | 2 +- include/llvm/IR/ConstantFolder.h | 2 +- include/llvm/IR/Constants.h | 13 +- include/llvm/IR/IRBuilder.h | 24 ++- include/llvm/IR/IRBuilderFolder.h | 2 +- include/llvm/IR/Instructions.h | 86 +++++----- include/llvm/IR/NoFolder.h | 2 +- include/llvm/IR/PatternMatch.h | 71 +++++++- lib/Analysis/ConstantFolding.cpp | 3 +- lib/Analysis/InstructionSimplify.cpp | 47 +++--- lib/Analysis/TargetTransformInfo.cpp | 4 +- lib/Analysis/ValueTracking.cpp | 6 +- lib/Analysis/VectorUtils.cpp | 8 +- lib/AsmParser/LLParser.cpp | 5 +- lib/Bitcode/Reader/BitcodeReader.cpp | 52 ++++-- lib/Bitcode/Writer/BitcodeWriter.cpp | 5 +- lib/Bitcode/Writer/ValueEnumerator.cpp | 43 ++++- lib/CodeGen/CodeGenPrepare.cpp | 4 +- lib/CodeGen/GlobalISel/IRTranslator.cpp | 7 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 11 +- lib/ExecutionEngine/Interpreter/Execution.cpp | 9 +- lib/IR/AsmWriter.cpp | 32 ++++ lib/IR/AutoUpgrade.cpp | 9 +- lib/IR/ConstantFold.cpp | 23 ++- lib/IR/ConstantFold.h | 2 +- lib/IR/Constants.cpp | 29 ++-- lib/IR/ConstantsContext.h | 86 ++++++---- lib/IR/Core.cpp | 4 +- lib/IR/Instruction.cpp | 3 + lib/IR/Instructions.cpp | 153 ++++++++++++------ lib/IR/Verifier.cpp | 2 +- lib/Target/AArch64/AArch64ISelLowering.cpp | 6 +- .../AMDGPU/AMDGPULowerKernelArguments.cpp | 2 +- .../AMDGPU/AMDGPURewriteOutArguments.cpp | 2 +- lib/Target/ARM/ARMISelLowering.cpp | 2 +- lib/Target/ARM/MVETailPredication.cpp | 4 +- lib/Target/X86/X86PartialReduction.cpp | 2 +- .../InstCombine/InstCombineCasts.cpp | 6 +- .../InstCombine/InstCombineCompares.cpp | 24 +-- .../InstCombineSimplifyDemanded.cpp | 11 +- .../InstCombine/InstCombineVectorOps.cpp | 124 +++++++------- .../InstCombine/InstructionCombining.cpp | 47 +++--- .../Instrumentation/MemorySanitizer.cpp | 3 +- lib/Transforms/Scalar/GVN.cpp | 14 +- lib/Transforms/Scalar/GVNSink.cpp | 12 +- lib/Transforms/Scalar/NewGVN.cpp | 4 +- .../Scalar/RewriteStatepointsForGC.cpp | 2 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- unittests/IR/PatternMatch.cpp | 17 +- 52 files changed, 672 insertions(+), 383 deletions(-) diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h index 5cba73e67f0..d11f8b5b44e 100644 --- a/include/llvm/ADT/ArrayRef.h +++ b/include/llvm/ADT/ArrayRef.h @@ -532,11 +532,21 @@ namespace llvm { return LHS.equals(RHS); } - template + template + inline bool operator==(SmallVectorImpl &LHS, ArrayRef RHS) { + return ArrayRef(LHS).equals(RHS); + } + + template inline bool operator!=(ArrayRef LHS, ArrayRef RHS) { return !(LHS == RHS); } + template + inline bool operator!=(SmallVectorImpl &LHS, ArrayRef RHS) { + return !(LHS == RHS); + } + /// @} template hash_code hash_value(ArrayRef S) { diff --git a/include/llvm/Analysis/ConstantFolding.h b/include/llvm/Analysis/ConstantFolding.h index 90dc14b0ed9..68aa1948ab8 100644 --- a/include/llvm/Analysis/ConstantFolding.h +++ b/include/llvm/Analysis/ConstantFolding.h @@ -119,10 +119,11 @@ Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx); /// Attempt to constant fold a shufflevector instruction with the -/// specified operands and indices. The constant result is returned if -/// successful; if not, null is returned. +/// specified operands and mask. See class ShuffleVectorInst for a description +/// of the mask representation. The constant result is returned if successful; +/// if not, null is returned. Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, - Constant *Mask); + ArrayRef Mask); /// ConstantFoldLoadFromConstPtr - Return the value that a load from C would /// produce if it is constant and determinable. If this is not determinable, diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h index b661caee684..7a9a1a81555 100644 --- a/include/llvm/Analysis/InstructionSimplify.h +++ b/include/llvm/Analysis/InstructionSimplify.h @@ -230,7 +230,8 @@ Value *SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, const SimplifyQuery &Q); /// Given operands for a ShuffleVectorInst, fold the result or return null. -Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, +/// See class ShuffleVectorInst for a description of the mask representation. +Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef Mask, Type *RetTy, const SimplifyQuery &Q); //=== Helper functions for higher up the class hierarchy. diff --git a/include/llvm/Analysis/TargetFolder.h b/include/llvm/Analysis/TargetFolder.h index 7277a1230e2..b23316ac3d9 100644 --- a/include/llvm/Analysis/TargetFolder.h +++ b/include/llvm/Analysis/TargetFolder.h @@ -259,7 +259,7 @@ public: } Constant *CreateShuffleVector(Constant *V1, Constant *V2, - Constant *Mask) const override { + ArrayRef Mask) const override { return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask)); } diff --git a/include/llvm/IR/ConstantFolder.h b/include/llvm/IR/ConstantFolder.h index 32939e79cd1..da4a18e3c18 100644 --- a/include/llvm/IR/ConstantFolder.h +++ b/include/llvm/IR/ConstantFolder.h @@ -265,7 +265,7 @@ public: } Constant *CreateShuffleVector(Constant *V1, Constant *V2, - Constant *Mask) const override { + ArrayRef Mask) const override { return ConstantExpr::getShuffleVector(V1, V2, Mask); } diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h index a345795ff54..868b038b055 100644 --- a/include/llvm/IR/Constants.h +++ b/include/llvm/IR/Constants.h @@ -1206,7 +1206,8 @@ public: Type *OnlyIfReducedTy = nullptr); static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx, Type *OnlyIfReducedTy = nullptr); - static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask, + static Constant *getShuffleVector(Constant *V1, Constant *V2, + ArrayRef Mask, Type *OnlyIfReducedTy = nullptr); static Constant *getExtractValue(Constant *Agg, ArrayRef Idxs, Type *OnlyIfReducedTy = nullptr); @@ -1225,6 +1226,16 @@ public: /// expression and return the list of indices. ArrayRef getIndices() const; + /// Assert that this is a shufflevector and return the mask. See class + /// ShuffleVectorInst for a description of the mask representation. + ArrayRef getShuffleMask() const; + + /// Assert that this is a shufflevector and return the mask. + /// + /// TODO: This is a temporary hack until we update the bitcode format for + /// shufflevector. + Constant *getShuffleMaskForBitcode() const; + /// Return a string representation for an opcode. const char *getOpcodeName() const; diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index 3b0ca0c1ae5..d4449e19600 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -2617,17 +2617,25 @@ public: Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name = "") { - if (auto *V1C = dyn_cast(V1)) - if (auto *V2C = dyn_cast(V2)) - if (auto *MC = dyn_cast(Mask)) - return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name); - return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); + SmallVector IntMask; + ShuffleVectorInst::getShuffleMask(cast(Mask), IntMask); + return CreateShuffleVector(V1, V2, IntMask, Name); } - Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef IntMask, + Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef Mask, const Twine &Name = "") { - Value *Mask = ConstantDataVector::get(Context, IntMask); - return CreateShuffleVector(V1, V2, Mask, Name); + SmallVector IntMask; + IntMask.assign(Mask.begin(), Mask.end()); + return CreateShuffleVector(V1, V2, IntMask, Name); + } + + /// See class ShuffleVectorInst for a description of the mask representation. + Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef Mask, + const Twine &Name = "") { + if (auto *V1C = dyn_cast(V1)) + if (auto *V2C = dyn_cast(V2)) + return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name); + return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); } Value *CreateExtractValue(Value *Agg, diff --git a/include/llvm/IR/IRBuilderFolder.h b/include/llvm/IR/IRBuilderFolder.h index a77a5213dce..e781e8e094a 100644 --- a/include/llvm/IR/IRBuilderFolder.h +++ b/include/llvm/IR/IRBuilderFolder.h @@ -129,7 +129,7 @@ public: virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt, Constant *Idx) const = 0; virtual Value *CreateShuffleVector(Constant *V1, Constant *V2, - Constant *Mask) const = 0; + ArrayRef Mask) const = 0; virtual Value *CreateExtractValue(Constant *Agg, ArrayRef IdxList) const = 0; virtual Value *CreateInsertValue(Constant *Agg, Constant *Val, diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index 713624d13be..857b7374ac4 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -1988,10 +1988,22 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value) // ShuffleVectorInst Class //===----------------------------------------------------------------------===// +constexpr int UndefMaskElem = -1; + /// This instruction constructs a fixed permutation of two /// input vectors. /// +/// For each element of the result vector, the shuffle mask selects an element +/// from one of the input vectors to copy to the result. Non-negative elements +/// in the mask represent an index into the concatenated pair of input vectors. +/// UndefMaskElem (-1) specifies that the result element is undefined. +/// +/// For scalable vectors, all the elements of the mask must be 0 or -1. This +/// requirement may be relaxed in the future. class ShuffleVectorInst : public Instruction { + SmallVector ShuffleMask; + Constant *ShuffleMaskForBitcode; + protected: // Note: Instruction needs to be a friend here to call cloneImpl. friend class Instruction; @@ -2004,13 +2016,15 @@ public: Instruction *InsertBefor = nullptr); ShuffleVectorInst(Value *V1, Value *V2, Value *Mask, const Twine &NameStr, BasicBlock *InsertAtEnd); + ShuffleVectorInst(Value *V1, Value *V2, ArrayRef Mask, + const Twine &NameStr = "", + Instruction *InsertBefor = nullptr); + ShuffleVectorInst(Value *V1, Value *V2, ArrayRef Mask, + const Twine &NameStr, BasicBlock *InsertAtEnd); - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } + void *operator new(size_t s) { return User::operator new(s, 2); } - /// Swap the first 2 operands and adjust the mask to preserve the semantics + /// Swap the operands and adjust the mask to preserve the semantics /// of the instruction. void commute(); @@ -2018,6 +2032,8 @@ public: /// formed with the specified operands. static bool isValidOperands(const Value *V1, const Value *V2, const Value *Mask); + static bool isValidOperands(const Value *V1, const Value *V2, + ArrayRef Mask); /// Overload to return most specific vector type. /// @@ -2028,36 +2044,33 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - Constant *getMask() const { - return cast(getOperand(2)); - } - - /// Return the shuffle mask value for the specified element of the mask. - /// Return -1 if the element is undef. - static int getMaskValue(const Constant *Mask, unsigned Elt); - /// Return the shuffle mask value of this instruction for the given element - /// index. Return -1 if the element is undef. - int getMaskValue(unsigned Elt) const { - return getMaskValue(getMask(), Elt); - } + /// index. Return UndefMaskElem if the element is undef. + int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; } /// Convert the input shuffle mask operand to a vector of integers. Undefined - /// elements of the mask are returned as -1. + /// elements of the mask are returned as UndefMaskElem. static void getShuffleMask(const Constant *Mask, SmallVectorImpl &Result); /// Return the mask for this instruction as a vector of integers. Undefined - /// elements of the mask are returned as -1. + /// elements of the mask are returned as UndefMaskElem. void getShuffleMask(SmallVectorImpl &Result) const { - return getShuffleMask(getMask(), Result); + Result.assign(ShuffleMask.begin(), ShuffleMask.end()); } - SmallVector getShuffleMask() const { - SmallVector Mask; - getShuffleMask(Mask); - return Mask; - } + /// Return the mask for this instruction, for use in bitcode. + /// + /// TODO: This is temporary until we decide a new bitcode encoding for + /// shufflevector. + Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; } + + static Constant *convertShuffleMaskForBitcode(ArrayRef Mask, + Type *ResultTy); + + void setShuffleMask(ArrayRef Mask); + + ArrayRef getShuffleMask() const { return ShuffleMask; } /// Return true if this shuffle returns a vector with a different number of /// elements than its source vectors. @@ -2065,7 +2078,7 @@ public: /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5> bool changesLength() const { unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements(); - unsigned NumMaskElts = getMask()->getType()->getVectorNumElements(); + unsigned NumMaskElts = ShuffleMask.size(); return NumSourceElts != NumMaskElts; } @@ -2074,7 +2087,7 @@ public: /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3> bool increasesLength() const { unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements(); - unsigned NumMaskElts = getMask()->getType()->getVectorNumElements(); + unsigned NumMaskElts = ShuffleMask.size(); return NumSourceElts < NumMaskElts; } @@ -2095,7 +2108,7 @@ public: /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3> /// TODO: Optionally allow length-changing shuffles. bool isSingleSource() const { - return !changesLength() && isSingleSourceMask(getMask()); + return !changesLength() && isSingleSourceMask(ShuffleMask); } /// Return true if this shuffle mask chooses elements from exactly one source @@ -2116,7 +2129,7 @@ public: /// from its input vectors. /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef> bool isIdentity() const { - return !changesLength() && isIdentityMask(getShuffleMask()); + return !changesLength() && isIdentityMask(ShuffleMask); } /// Return true if this shuffle lengthens exactly one source vector with @@ -2157,7 +2170,7 @@ public: /// In that case, the shuffle is better classified as an identity shuffle. /// TODO: Optionally allow length-changing shuffles. bool isSelect() const { - return !changesLength() && isSelectMask(getMask()); + return !changesLength() && isSelectMask(ShuffleMask); } /// Return true if this shuffle mask swaps the order of elements from exactly @@ -2177,7 +2190,7 @@ public: /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef> /// TODO: Optionally allow length-changing shuffles. bool isReverse() const { - return !changesLength() && isReverseMask(getMask()); + return !changesLength() && isReverseMask(ShuffleMask); } /// Return true if this shuffle mask chooses all elements with the same value @@ -2199,7 +2212,7 @@ public: /// TODO: Optionally allow length-changing shuffles. /// TODO: Optionally allow splats from other elements. bool isZeroEltSplat() const { - return !changesLength() && isZeroEltSplatMask(getMask()); + return !changesLength() && isZeroEltSplatMask(ShuffleMask); } /// Return true if this shuffle mask is a transpose mask. @@ -2248,7 +2261,7 @@ public: /// exact specification. /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6> bool isTranspose() const { - return !changesLength() && isTransposeMask(getMask()); + return !changesLength() && isTransposeMask(ShuffleMask); } /// Return true if this shuffle mask is an extract subvector mask. @@ -2267,7 +2280,7 @@ public: /// Return true if this shuffle mask is an extract subvector mask. bool isExtractSubvectorMask(int &Index) const { int NumSrcElts = Op<0>()->getType()->getVectorNumElements(); - return isExtractSubvectorMask(getMask(), NumSrcElts, Index); + return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index); } /// Change values in a shuffle permute mask assuming the two vector operands @@ -2293,9 +2306,8 @@ public: }; template <> -struct OperandTraits : - public FixedNumOperandTraits { -}; +struct OperandTraits + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value) diff --git a/include/llvm/IR/NoFolder.h b/include/llvm/IR/NoFolder.h index 1e35cfe096e..dcffa6b2f9d 100644 --- a/include/llvm/IR/NoFolder.h +++ b/include/llvm/IR/NoFolder.h @@ -300,7 +300,7 @@ public: } Instruction *CreateShuffleVector(Constant *V1, Constant *V2, - Constant *Mask) const override { + ArrayRef Mask) const override { return new ShuffleVectorInst(V1, V2, Mask); } diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h index 03b92e524ad..30c203dedf6 100644 --- a/include/llvm/IR/PatternMatch.h +++ b/include/llvm/IR/PatternMatch.h @@ -50,6 +50,10 @@ template bool match(Val *V, const Pattern &P) { return const_cast(P).match(V); } +template bool match(ArrayRef Mask, const Pattern &P) { + return const_cast(P).match(Mask); +} + template struct OneUse_match { SubPattern_t SubPattern; @@ -1350,12 +1354,69 @@ m_ExtractElement(const Val_t &Val, const Idx_t &Idx) { return TwoOps_match(Val, Idx); } -/// Matches ShuffleVectorInst. +/// Matches shuffle. +template struct Shuffle_match { + T0 Op1; + T1 Op2; + T2 Mask; + + Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) + : Op1(Op1), Op2(Op2), Mask(Mask) {} + + template bool match(OpTy *V) { + if (auto *I = dyn_cast(V)) { + return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && + Mask.match(I->getShuffleMask()); + } + return false; + } +}; + +struct m_Mask { + ArrayRef &MaskRef; + m_Mask(ArrayRef &MaskRef) : MaskRef(MaskRef) {} + bool match(ArrayRef Mask) { + MaskRef = Mask; + return true; + } +}; + +struct m_ZeroMask { + bool match(ArrayRef Mask) { + return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); + } +}; + +struct m_SpecificMask { + ArrayRef &MaskRef; + m_SpecificMask(ArrayRef &MaskRef) : MaskRef(MaskRef) {} + bool match(ArrayRef Mask) { return MaskRef == Mask; } +}; + +struct m_SplatOrUndefMask { + int &SplatIndex; + m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} + bool match(ArrayRef Mask) { + auto First = find_if(Mask, [](int Elem) { return Elem != -1; }); + if (First == Mask.end()) + return false; + SplatIndex = *First; + return all_of(Mask, + [First](int Elem) { return Elem == *First || Elem == -1; }); + } +}; + +/// Matches ShuffleVectorInst independently of mask value. +template +inline TwoOps_match +m_ShuffleVector(const V1_t &v1, const V2_t &v2) { + return TwoOps_match(v1, v2); +} + template -inline ThreeOps_match -m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) { - return ThreeOps_match(v1, v2, - m); +inline Shuffle_match +m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { + return Shuffle_match(v1, v2, mask); } /// Matches LoadInst. diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp index 5efebe2937e..721ba735e9d 100644 --- a/lib/Analysis/ConstantFolding.cpp +++ b/lib/Analysis/ConstantFolding.cpp @@ -1066,7 +1066,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode, case Instruction::InsertElement: return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]); case Instruction::ShuffleVector: - return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]); + return ConstantExpr::getShuffleVector( + Ops[0], Ops[1], cast(InstOrCE)->getShuffleMask()); } } diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp index e62ddf793a1..086ff743354 100644 --- a/lib/Analysis/InstructionSimplify.cpp +++ b/lib/Analysis/InstructionSimplify.cpp @@ -4439,36 +4439,31 @@ static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1, return RootVec; } -static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, - Type *RetTy, const SimplifyQuery &Q, +static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, + ArrayRef Mask, Type *RetTy, + const SimplifyQuery &Q, unsigned MaxRecurse) { - if (isa(Mask)) + if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; })) return UndefValue::get(RetTy); Type *InVecTy = Op0->getType(); - ElementCount MaskEltCount = Mask->getType()->getVectorElementCount(); + unsigned MaskNumElts = Mask.size(); ElementCount InVecEltCount = InVecTy->getVectorElementCount(); - assert(MaskEltCount.Scalable == InVecEltCount.Scalable && - "vscale mismatch between input vector and mask"); - - bool Scalable = MaskEltCount.Scalable; + bool Scalable = InVecEltCount.Scalable; SmallVector Indices; - if (!Scalable) { - ShuffleVectorInst::getShuffleMask(Mask, Indices); - assert(MaskEltCount.Min == Indices.size() && - "Size of Indices not same as number of mask elements?"); - } + Indices.assign(Mask.begin(), Mask.end()); + // Canonicalization: If mask does not select elements from an input vector, + // replace that input vector with undef. if (!Scalable) { - // Canonicalization: If mask does not select elements from an input vector, - // replace that input vector with undef. bool MaskSelects0 = false, MaskSelects1 = false; - for (unsigned i = 0; i != MaskEltCount.Min; ++i) { + unsigned InVecNumElts = InVecEltCount.Min; + for (unsigned i = 0; i != MaskNumElts; ++i) { if (Indices[i] == -1) continue; - if ((unsigned)Indices[i] < InVecEltCount.Min) + if ((unsigned)Indices[i] < InVecNumElts) MaskSelects0 = true; else MaskSelects1 = true; @@ -4514,8 +4509,8 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, assert(isa(Op1) && "Expected undef operand 1 for splat"); // Shuffle mask undefs become undefined constant result elements. - SmallVector VecC(MaskEltCount.Min, C); - for (unsigned i = 0; i != MaskEltCount.Min; ++i) + SmallVector VecC(MaskNumElts, C); + for (unsigned i = 0; i != MaskNumElts; ++i) if (Indices[i] == -1) VecC[i] = UndefValue::get(C->getType()); return ConstantVector::get(VecC); @@ -4526,7 +4521,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, // value type is same as the input vectors' type. if (auto *OpShuf = dyn_cast(Op0)) if (isa(Op1) && RetTy == InVecTy && - OpShuf->getMask()->getSplatValue()) + is_splat(OpShuf->getShuffleMask())) return Op0; // All remaining transformation depend on the value of the mask, which is @@ -4545,7 +4540,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, // shuffle. This handles simple identity shuffles as well as chains of // shuffles that may widen/narrow and/or move elements across lanes and back. Value *RootVec = nullptr; - for (unsigned i = 0; i != MaskEltCount.Min; ++i) { + for (unsigned i = 0; i != MaskNumElts; ++i) { // Note that recursion is limited for each vector element, so if any element // exceeds the limit, this will fail to simplify. RootVec = @@ -4559,8 +4554,9 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, } /// Given operands for a ShuffleVectorInst, fold the result or return null. -Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, - Type *RetTy, const SimplifyQuery &Q) { +Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, + ArrayRef Mask, Type *RetTy, + const SimplifyQuery &Q) { return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit); } @@ -5523,8 +5519,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ, } case Instruction::ShuffleVector: { auto *SVI = cast(I); - Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), - SVI->getMask(), SVI->getType(), Q); + Result = + SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), + SVI->getShuffleMask(), SVI->getType(), Q); break; } case Instruction::PHI: diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp index 89b1783a9a0..2ae44caaaa3 100644 --- a/lib/Analysis/TargetTransformInfo.cpp +++ b/lib/Analysis/TargetTransformInfo.cpp @@ -888,7 +888,7 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft, for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2) Mask[i] = val; - SmallVector ActualMask = SI->getShuffleMask(); + ArrayRef ActualMask = SI->getShuffleMask(); return Mask == ActualMask; } @@ -1153,7 +1153,7 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, // Fill the rest of the mask with -1 for undef. std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1); - SmallVector Mask = Shuffle->getShuffleMask(); + ArrayRef Mask = Shuffle->getShuffleMask(); if (ShuffleMask != Mask) return RK_None; diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp index 00751302076..6a4d596a3b7 100644 --- a/lib/Analysis/ValueTracking.cpp +++ b/lib/Analysis/ValueTracking.cpp @@ -168,16 +168,16 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf, APInt &DemandedLHS, APInt &DemandedRHS) { // The length of scalable vectors is unknown at compile time, thus we // cannot check their values - if (Shuf->getMask()->getType()->getVectorElementCount().Scalable) + if (Shuf->getType()->getVectorElementCount().Scalable) return false; int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements(); - int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements(); + int NumMaskElts = Shuf->getType()->getVectorNumElements(); DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts); if (DemandedElts.isNullValue()) return true; // Simple case of a shuffle with zeroinitializer. - if (isa(Shuf->getMask())) { + if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) { DemandedLHS.setBit(0); return true; } diff --git a/lib/Analysis/VectorUtils.cpp b/lib/Analysis/VectorUtils.cpp index 532518acc6a..c8ca2052919 100644 --- a/lib/Analysis/VectorUtils.cpp +++ b/lib/Analysis/VectorUtils.cpp @@ -339,9 +339,9 @@ const llvm::Value *llvm::getSplatValue(const Value *V) { // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...> Value *Splat; - if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat), - m_ZeroInt()), - m_Value(), m_ZeroInt()))) + if (match(V, m_ShuffleVector( + m_InsertElement(m_Value(), m_Value(Splat), m_ZeroInt()), + m_Value(), m_ZeroMask()))) return Splat; return nullptr; @@ -366,7 +366,7 @@ bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) { if (auto *Shuf = dyn_cast(V)) { // FIXME: We can safely allow undefs here. If Index was specified, we will // check that the mask elt is defined at the required index. - if (!Shuf->getMask()->getSplatValue()) + if (!is_splat(Shuf->getShuffleMask())) return false; // Match any index. diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index d219f607410..ebb17204864 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -3636,8 +3636,9 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { return Error(ID.Loc, "expected three operands to shufflevector"); if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2])) return Error(ID.Loc, "invalid operands to shufflevector"); - ID.ConstantVal = - ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]); + SmallVector Mask; + ShuffleVectorInst::getShuffleMask(cast(Elts[2]), Mask); + ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask); } else if (Opc == Instruction::ExtractElement) { if (Elts.size() != 2) return Error(ID.Loc, "expected two operands to extractelement"); diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index de2a6b47e6e..16d3b79598f 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2337,6 +2337,15 @@ Error BitcodeReader::parseConstants() { Type *CurFullTy = Type::getInt32Ty(Context); unsigned NextCstNo = ValueList.size(); + struct DelayedShufTy { + VectorType *OpTy; + VectorType *RTy; + Type *CurFullTy; + uint64_t Op0Idx; + uint64_t Op1Idx; + uint64_t Op2Idx; + }; + std::vector DelayedShuffles; while (true) { Expected MaybeEntry = Stream.advanceSkippingSubblocks(); if (!MaybeEntry) @@ -2353,6 +2362,29 @@ Error BitcodeReader::parseConstants() { // Once all the constants have been read, go through and resolve forward // references. + // + // We have to treat shuffles specially because they don't have three + // operands anymore. We need to convert the shuffle mask into an array, + // and we can't convert a forward reference. + for (auto &DelayedShuffle : DelayedShuffles) { + VectorType *OpTy = DelayedShuffle.OpTy; + VectorType *RTy = DelayedShuffle.RTy; + uint64_t Op0Idx = DelayedShuffle.Op0Idx; + uint64_t Op1Idx = DelayedShuffle.Op1Idx; + uint64_t Op2Idx = DelayedShuffle.Op2Idx; + Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy); + Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy); + Type *ShufTy = + VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount()); + Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy); + if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2)) + return error("Invalid shufflevector operands"); + SmallVector Mask; + ShuffleVectorInst::getShuffleMask(Op2, Mask); + Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask); + ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy); + ++NextCstNo; + } ValueList.resolveConstantForwardRefs(); return Error::success(); case BitstreamEntry::Record: @@ -2694,13 +2726,9 @@ Error BitcodeReader::parseConstants() { VectorType *OpTy = dyn_cast(CurTy); if (Record.size() < 3 || !OpTy) return error("Invalid record"); - Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy); - Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy); - Type *ShufTy = VectorType::get(Type::getInt32Ty(Context), - OpTy->getElementCount()); - Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy); - V = ConstantExpr::getShuffleVector(Op0, Op1, Op2); - break; + DelayedShuffles.push_back( + {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]}); + continue; } case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval] VectorType *RTy = dyn_cast(CurTy); @@ -2708,13 +2736,9 @@ Error BitcodeReader::parseConstants() { dyn_cast_or_null(getTypeByID(Record[0])); if (Record.size() < 4 || !RTy || !OpTy) return error("Invalid record"); - Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy); - Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy); - Type *ShufTy = VectorType::get(Type::getInt32Ty(Context), - RTy->getElementCount()); - Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy); - V = ConstantExpr::getShuffleVector(Op0, Op1, Op2); - break; + DelayedShuffles.push_back( + {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]}); + continue; } case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred] if (Record.size() < 4) diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index a27eada55d7..e9ad00d6d4b 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -2515,7 +2515,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, } Record.push_back(VE.getValueID(C->getOperand(0))); Record.push_back(VE.getValueID(C->getOperand(1))); - Record.push_back(VE.getValueID(C->getOperand(2))); + Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode())); break; case Instruction::ICmp: case Instruction::FCmp: @@ -2702,7 +2702,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_SHUFFLEVEC; pushValueAndType(I.getOperand(0), InstID, Vals); pushValue(I.getOperand(1), InstID, Vals); - pushValue(I.getOperand(2), InstID, Vals); + pushValue(cast(I).getShuffleMaskForBitcode(), InstID, + Vals); break; case Instruction::ICmp: case Instruction::FCmp: { diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index f59c906c7b7..7419c4687fa 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -88,11 +88,16 @@ static void orderValue(const Value *V, OrderMap &OM) { if (OM.lookup(V).first) return; - if (const Constant *C = dyn_cast(V)) - if (C->getNumOperands() && !isa(C)) + if (const Constant *C = dyn_cast(V)) { + if (C->getNumOperands() && !isa(C)) { for (const Value *Op : C->operands()) if (!isa(Op) && !isa(Op)) orderValue(Op, OM); + if (auto *CE = dyn_cast(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + orderValue(CE->getShuffleMaskForBitcode(), OM); + } + } // Note: we cannot cache this lookup above, since inserting into the map // changes the map's size, and thus affects the other IDs. @@ -155,11 +160,14 @@ static OrderMap orderModule(const Module &M) { for (const Argument &A : F.args()) orderValue(&A, OM); for (const BasicBlock &BB : F) - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Value *Op : I.operands()) if ((isa(*Op) && !isa(*Op)) || isa(*Op)) orderValue(Op, OM); + if (auto *SVI = dyn_cast(&I)) + orderValue(SVI->getShuffleMaskForBitcode(), OM); + } for (const BasicBlock &BB : F) for (const Instruction &I : BB) orderValue(&I, OM); @@ -250,11 +258,17 @@ static void predictValueUseListOrder(const Value *V, const Function *F, predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack); // Recursive descent into constants. - if (const Constant *C = dyn_cast(V)) - if (C->getNumOperands()) // Visit GlobalValues. + if (const Constant *C = dyn_cast(V)) { + if (C->getNumOperands()) { // Visit GlobalValues. for (const Value *Op : C->operands()) if (isa(Op)) // Visit GlobalValues. predictValueUseListOrder(Op, F, OM, Stack); + if (auto *CE = dyn_cast(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM, + Stack); + } + } } static UseListOrderStack predictUseListOrder(const Module &M) { @@ -279,10 +293,14 @@ static UseListOrderStack predictUseListOrder(const Module &M) { for (const Argument &A : F.args()) predictValueUseListOrder(&A, &F, OM, Stack); for (const BasicBlock &BB : F) - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Value *Op : I.operands()) if (isa(*Op) || isa(*Op)) // Visit GlobalValues. predictValueUseListOrder(Op, &F, OM, Stack); + if (auto *SVI = dyn_cast(&I)) + predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM, + Stack); + } for (const BasicBlock &BB : F) for (const Instruction &I : BB) predictValueUseListOrder(&I, &F, OM, Stack); @@ -413,6 +431,8 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateMetadata(&F, MD->getMetadata()); } + if (auto *SVI = dyn_cast(&I)) + EnumerateType(SVI->getShuffleMaskForBitcode()->getType()); EnumerateType(I.getType()); if (const auto *Call = dyn_cast(&I)) EnumerateAttributes(Call->getAttributes()); @@ -836,6 +856,9 @@ void ValueEnumerator::EnumerateValue(const Value *V) { I != E; ++I) if (!isa(*I)) // Don't enumerate BB operand to BlockAddress. EnumerateValue(*I); + if (auto *CE = dyn_cast(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + EnumerateValue(CE->getShuffleMaskForBitcode()); // Finally, add the value. Doing this could make the ValueID reference be // dangling, don't reuse it. @@ -913,6 +936,9 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) { EnumerateOperandType(Op); } + if (auto *CE = dyn_cast(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + EnumerateOperandType(CE->getShuffleMaskForBitcode()); } void ValueEnumerator::EnumerateAttributes(AttributeList PAL) { @@ -958,11 +984,14 @@ void ValueEnumerator::incorporateFunction(const Function &F) { // Add all function-level constants to the value table. for (const BasicBlock &BB : F) { - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Use &OI : I.operands()) { if ((isa(OI) && !isa(OI)) || isa(OI)) EnumerateValue(OI); } + if (auto *SVI = dyn_cast(&I)) + EnumerateValue(SVI->getShuffleMaskForBitcode()); + } BasicBlocks.push_back(&BB); ValueMap[&BB] = BasicBlocks.size(); } diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index 4a416c3c823..1654eba6bc5 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -6257,7 +6257,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) { } static bool isBroadcastShuffle(ShuffleVectorInst *SVI) { - SmallVector Mask(SVI->getShuffleMask()); + ArrayRef Mask(SVI->getShuffleMask()); int SplatElem = -1; for (unsigned i = 0; i < Mask.size(); ++i) { if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem) @@ -6307,7 +6307,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) { assert(InsertPt != UserBB->end()); InsertedShuffle = new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), - SVI->getOperand(2), "", &*InsertPt); + SVI->getShuffleMask(), "", &*InsertPt); InsertedShuffle->setDebugLoc(SVI->getDebugLoc()); } diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index 58888c4dadc..8135ce86cd2 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1973,8 +1973,11 @@ bool IRTranslator::translateExtractElement(const User &U, bool IRTranslator::translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder) { - SmallVector Mask; - ShuffleVectorInst::getShuffleMask(cast(U.getOperand(2)), Mask); + ArrayRef Mask; + if (auto *SVI = dyn_cast(&U)) + Mask = SVI->getShuffleMask(); + else + Mask = cast(U).getShuffleMask(); ArrayRef MaskAlloc = MF->allocateShuffleMask(Mask); MIRBuilder .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)}, diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 26ed6e22af9..ed24e004f90 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3437,14 +3437,19 @@ void SelectionDAGBuilder::visitExtractElement(const User &I) { void SelectionDAGBuilder::visitShuffleVector(const User &I) { SDValue Src1 = getValue(I.getOperand(0)); SDValue Src2 = getValue(I.getOperand(1)); - Constant *MaskV = cast(I.getOperand(2)); + ArrayRef Mask; + if (auto *SVI = dyn_cast(&I)) + Mask = SVI->getShuffleMask(); + else + Mask = cast(I).getShuffleMask(); SDLoc DL = getCurSDLoc(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); EVT SrcVT = Src1.getValueType(); unsigned SrcNumElts = SrcVT.getVectorNumElements(); - if (MaskV->isNullValue() && VT.isScalableVector()) { + if (all_of(Mask, [](int Elem) { return Elem == 0; }) && + VT.isScalableVector()) { // Canonical splat form of first element of first input vector. SDValue FirstElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, @@ -3458,8 +3463,6 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { // for targets that support a SPLAT_VECTOR for non-scalable vector types. assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle"); - SmallVector Mask; - ShuffleVectorInst::getShuffleMask(MaskV, Mask); unsigned MaskNumElts = Mask.size(); if (SrcNumElts == MaskNumElts) { diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp index 51f31d3d5d8..4a5fb1feb18 100644 --- a/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -1868,7 +1868,6 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){ GenericValue Src1 = getOperandValue(I.getOperand(0), SF); GenericValue Src2 = getOperandValue(I.getOperand(1), SF); - GenericValue Src3 = getOperandValue(I.getOperand(2), SF); GenericValue Dest; // There is no need to check types of src1 and src2, because the compiled @@ -1878,7 +1877,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){ Type *TyContained = Ty->getElementType(); unsigned src1Size = (unsigned)Src1.AggregateVal.size(); unsigned src2Size = (unsigned)Src2.AggregateVal.size(); - unsigned src3Size = (unsigned)Src3.AggregateVal.size(); + unsigned src3Size = I.getShuffleMask().size(); Dest.AggregateVal.resize(src3Size); @@ -1888,7 +1887,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){ break; case Type::IntegerTyID: for( unsigned i=0; i(V) ? GlobalPrefix : LocalPrefix); } +static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef Mask) { + Out << ", <"; + if (Ty->getVectorIsScalable()) + Out << "vscale x "; + Out << Mask.size() << " x i32> "; + bool FirstElt = true; + if (all_of(Mask, [](int Elt) { return Elt == 0; })) { + Out << "zeroinitializer"; + } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + Out << "undef"; + } else { + Out << "<"; + for (int Elt : Mask) { + if (FirstElt) + FirstElt = false; + else + Out << ", "; + Out << "i32 "; + if (Elt == UndefMaskElem) + Out << "undef"; + else + Out << Elt; + } + Out << ">"; + } +} + namespace { class TypePrinting { @@ -1547,6 +1574,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, TypePrinter.print(CE->getType(), Out); } + if (CE->getOpcode() == Instruction::ShuffleVector) + PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask()); + Out << ')'; return; } @@ -4093,6 +4123,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) { RMWI->getSyncScopeID()); } else if (const FenceInst *FI = dyn_cast(&I)) { writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID()); + } else if (const ShuffleVectorInst *SVI = dyn_cast(&I)) { + PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask()); } // Print Metadata info. diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp index de5b94403bd..96021dc1fbd 100644 --- a/lib/IR/AutoUpgrade.cpp +++ b/lib/IR/AutoUpgrade.cpp @@ -2335,11 +2335,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { PointerType::getUnqual(VT)); Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1)); if (NumSrcElts == 2) - Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), - { 0, 1, 0, 1 }); + Rep = Builder.CreateShuffleVector( + Load, UndefValue::get(Load->getType()), ArrayRef{0, 1, 0, 1}); else - Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), - { 0, 1, 2, 3, 0, 1, 2, 3 }); + Rep = + Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), + ArrayRef{0, 1, 2, 3, 0, 1, 2, 3}); } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") || Name.startswith("avx512.mask.shuf.f"))) { unsigned Imm = cast(CI->getArgOperand(2))->getZExtValue(); diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp index 07292e50fc8..c4465d9bfa0 100644 --- a/lib/IR/ConstantFold.cpp +++ b/lib/IR/ConstantFold.cpp @@ -876,22 +876,22 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val, return ConstantVector::get(Result); } -Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, - Constant *V2, - Constant *Mask) { - ElementCount MaskEltCount = Mask->getType()->getVectorElementCount(); +Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, + ArrayRef Mask) { + unsigned MaskNumElts = Mask.size(); + ElementCount MaskEltCount = {MaskNumElts, + V1->getType()->getVectorIsScalable()}; Type *EltTy = V1->getType()->getVectorElementType(); // Undefined shuffle mask -> undefined value. - if (isa(Mask)) - return UndefValue::get(VectorType::get(EltTy, MaskEltCount)); - - // Don't break the bitcode reader hack. - if (isa(Mask)) return nullptr; + if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + return UndefValue::get(VectorType::get(EltTy, MaskNumElts)); + } // If the mask is all zeros this is a splat, no need to go through all // elements. - if (isa(Mask) && !MaskEltCount.Scalable) { + if (all_of(Mask, [](int Elt) { return Elt == 0; }) && + !MaskEltCount.Scalable) { Type *Ty = IntegerType::get(V1->getContext(), 32); Constant *Elt = ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0)); @@ -903,13 +903,12 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, if (ValTy->isScalable()) return nullptr; - unsigned MaskNumElts = MaskEltCount.Min; unsigned SrcNumElts = V1->getType()->getVectorNumElements(); // Loop over the shuffle mask, evaluating each element. SmallVector Result; for (unsigned i = 0; i != MaskNumElts; ++i) { - int Elt = ShuffleVectorInst::getMaskValue(Mask, i); + int Elt = Mask[i]; if (Elt == -1) { Result.push_back(UndefValue::get(EltTy)); continue; diff --git a/lib/IR/ConstantFold.h b/lib/IR/ConstantFold.h index 9ad6e14e9e4..0cdd5cf3cbc 100644 --- a/lib/IR/ConstantFold.h +++ b/lib/IR/ConstantFold.h @@ -38,7 +38,7 @@ template class ArrayRef; Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt, Constant *Idx); Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, - Constant *Mask); + ArrayRef Mask); Constant *ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef Idxs); Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp index e001b523075..60ab1a595c3 100644 --- a/lib/IR/Constants.cpp +++ b/lib/IR/Constants.cpp @@ -1230,8 +1230,7 @@ Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) { Constant *UndefV = UndefValue::get(VTy); V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0)); // Build shuffle mask to perform the splat. - Type *MaskTy = VectorType::get(I32Ty, EC); - Constant *Zeros = ConstantAggregateZero::get(MaskTy); + SmallVector Zeros(EC.Min, 0); // Splat. return ConstantExpr::getShuffleVector(V, UndefV, Zeros); } @@ -1298,6 +1297,14 @@ unsigned ConstantExpr::getPredicate() const { return cast(this)->predicate; } +ArrayRef ConstantExpr::getShuffleMask() const { + return cast(this)->ShuffleMask; +} + +Constant *ConstantExpr::getShuffleMaskForBitcode() const { + return cast(this)->ShuffleMaskForBitcode; +} + Constant * ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const { assert(Op->getType() == getOperand(OpNo)->getType() && @@ -1349,7 +1356,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef Ops, Type *Ty, case Instruction::ExtractValue: return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy); case Instruction::ShuffleVector: - return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2], + return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(), OnlyIfReducedTy); case Instruction::GetElementPtr: { auto *GEPO = cast(this); @@ -2152,7 +2159,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, if (InRangeIndex && *InRangeIndex < 63) SubClassOptionalData |= (*InRangeIndex + 1) << 1; const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0, - SubClassOptionalData, None, Ty); + SubClassOptionalData, None, None, Ty); LLVMContextImpl *pImpl = C->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ReqTy, Key); @@ -2254,23 +2261,25 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt, } Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2, - Constant *Mask, Type *OnlyIfReducedTy) { + ArrayRef Mask, + Type *OnlyIfReducedTy) { assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) && "Invalid shuffle vector constant expr operands!"); if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask)) return FC; // Fold a few common cases. - ElementCount NElts = Mask->getType()->getVectorElementCount(); + unsigned NElts = Mask.size(); Type *EltTy = V1->getType()->getVectorElementType(); - Type *ShufTy = VectorType::get(EltTy, NElts); + bool TypeIsScalable = V1->getType()->getVectorIsScalable(); + Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable); if (OnlyIfReducedTy == ShufTy) return nullptr; // Look up the constant in the table first to ensure uniqueness - Constant *ArgVec[] = { V1, V2, Mask }; - const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec); + Constant *ArgVec[] = {V1, V2}; + ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask); LLVMContextImpl *pImpl = ShufTy->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ShufTy, Key); @@ -3117,7 +3126,7 @@ Instruction *ConstantExpr::getAsInstruction() const { case Instruction::ExtractValue: return ExtractValueInst::Create(Ops[0], getIndices()); case Instruction::ShuffleVector: - return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]); + return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask()); case Instruction::GetElementPtr: { const auto *GO = cast(this); diff --git a/lib/IR/ConstantsContext.h b/lib/IR/ConstantsContext.h index 9ecbb613783..ad521c8a1bd 100644 --- a/lib/IR/ConstantsContext.h +++ b/lib/IR/ConstantsContext.h @@ -26,6 +26,7 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/OperandTraits.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" @@ -146,21 +147,24 @@ public: /// shufflevector constant exprs. class ShuffleVectorConstantExpr : public ConstantExpr { public: - ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3) - : ConstantExpr(VectorType::get( - cast(C1->getType())->getElementType(), - cast(C3->getType())->getElementCount()), - Instruction::ShuffleVector, - &Op<0>(), 3) { + ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef Mask) + : ConstantExpr( + VectorType::get(cast(C1->getType())->getElementType(), + Mask.size(), C1->getType()->getVectorIsScalable()), + Instruction::ShuffleVector, &Op<0>(), 2) { + assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) && + "Invalid shuffle vector instruction operands!"); Op<0>() = C1; Op<1>() = C2; - Op<2>() = C3; + ShuffleMask.assign(Mask.begin(), Mask.end()); + ShuffleMaskForBitcode = + ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType()); } - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } + SmallVector ShuffleMask; + Constant *ShuffleMaskForBitcode; + + void *operator new(size_t s) { return User::operator new(s, 2); } /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -319,7 +323,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value) template <> struct OperandTraits - : public FixedNumOperandTraits {}; + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value) template <> @@ -460,36 +464,58 @@ struct InlineAsmKeyType { }; struct ConstantExprKeyType { +private: uint8_t Opcode; uint8_t SubclassOptionalData; uint16_t SubclassData; ArrayRef Ops; ArrayRef Indexes; + ArrayRef ShuffleMask; Type *ExplicitTy; + static ArrayRef getShuffleMaskIfValid(const ConstantExpr *CE) { + if (CE->getOpcode() == Instruction::ShuffleVector) + return CE->getShuffleMask(); + return None; + } + + static ArrayRef getIndicesIfValid(const ConstantExpr *CE) { + if (CE->hasIndices()) + return CE->getIndices(); + return None; + } + + static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) { + if (auto *GEPCE = dyn_cast(CE)) + return GEPCE->getSourceElementType(); + return nullptr; + } + +public: ConstantExprKeyType(unsigned Opcode, ArrayRef Ops, unsigned short SubclassData = 0, unsigned short SubclassOptionalData = 0, ArrayRef Indexes = None, + ArrayRef ShuffleMask = None, Type *ExplicitTy = nullptr) : Opcode(Opcode), SubclassOptionalData(SubclassOptionalData), SubclassData(SubclassData), Ops(Ops), Indexes(Indexes), - ExplicitTy(ExplicitTy) {} + ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {} ConstantExprKeyType(ArrayRef Operands, const ConstantExpr *CE) : Opcode(CE->getOpcode()), SubclassOptionalData(CE->getRawSubclassOptionalData()), SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands), - Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef()), - ExplicitTy(nullptr) {} + Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)), + ExplicitTy(getSourceElementTypeIfValid(CE)) {} ConstantExprKeyType(const ConstantExpr *CE, SmallVectorImpl &Storage) : Opcode(CE->getOpcode()), SubclassOptionalData(CE->getRawSubclassOptionalData()), SubclassData(CE->isCompare() ? CE->getPredicate() : 0), - Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef()), - ExplicitTy(nullptr) { + Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)), + ExplicitTy(getSourceElementTypeIfValid(CE)) { assert(Storage.empty() && "Expected empty storage"); for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I) Storage.push_back(CE->getOperand(I)); @@ -499,7 +525,8 @@ struct ConstantExprKeyType { bool operator==(const ConstantExprKeyType &X) const { return Opcode == X.Opcode && SubclassData == X.SubclassData && SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops && - Indexes == X.Indexes; + Indexes == X.Indexes && ShuffleMask == X.ShuffleMask && + ExplicitTy == X.ExplicitTy; } bool operator==(const ConstantExpr *CE) const { @@ -514,15 +541,21 @@ struct ConstantExprKeyType { for (unsigned I = 0, E = Ops.size(); I != E; ++I) if (Ops[I] != CE->getOperand(I)) return false; - if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef())) + if (Indexes != getIndicesIfValid(CE)) + return false; + if (ShuffleMask != getShuffleMaskIfValid(CE)) + return false; + if (ExplicitTy != getSourceElementTypeIfValid(CE)) return false; return true; } unsigned getHash() const { - return hash_combine(Opcode, SubclassOptionalData, SubclassData, - hash_combine_range(Ops.begin(), Ops.end()), - hash_combine_range(Indexes.begin(), Indexes.end())); + return hash_combine( + Opcode, SubclassOptionalData, SubclassData, + hash_combine_range(Ops.begin(), Ops.end()), + hash_combine_range(Indexes.begin(), Indexes.end()), + hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy); } using TypeClass = ConstantInfo::TypeClass; @@ -546,17 +579,14 @@ struct ConstantExprKeyType { case Instruction::InsertElement: return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]); case Instruction::ShuffleVector: - return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]); + return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask); case Instruction::InsertValue: return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty); case Instruction::ExtractValue: return new ExtractValueConstantExpr(Ops[0], Indexes, Ty); case Instruction::GetElementPtr: - return GetElementPtrConstantExpr::Create( - ExplicitTy ? ExplicitTy - : cast(Ops[0]->getType()->getScalarType()) - ->getElementType(), - Ops[0], Ops.slice(1), Ty, SubclassOptionalData); + return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1), + Ty, SubclassOptionalData); case Instruction::ICmp: return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData, Ops[0], Ops[1]); diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp index 313bc65e775..da2e25e7797 100644 --- a/lib/IR/Core.cpp +++ b/lib/IR/Core.cpp @@ -1781,9 +1781,11 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant, LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant, LLVMValueRef VectorBConstant, LLVMValueRef MaskConstant) { + SmallVector IntMask; + ShuffleVectorInst::getShuffleMask(unwrap(MaskConstant), IntMask); return wrap(ConstantExpr::getShuffleVector(unwrap(VectorAConstant), unwrap(VectorBConstant), - unwrap(MaskConstant))); + IntMask)); } LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList, diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index 2ef109067dd..cd476a36015 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -443,6 +443,9 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, RMWI->isVolatile() == cast(I2)->isVolatile() && RMWI->getOrdering() == cast(I2)->getOrdering() && RMWI->getSyncScopeID() == cast(I2)->getSyncScopeID(); + if (const ShuffleVectorInst *SVI = dyn_cast(I1)) + return SVI->getShuffleMask() == + cast(I2)->getShuffleMask(); return true; } diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index 3807752eae0..37e194a0fea 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -1843,57 +1843,109 @@ bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt, ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask, const Twine &Name, Instruction *InsertBefore) -: Instruction(VectorType::get(cast(V1->getType())->getElementType(), - cast(Mask->getType())->getElementCount()), - ShuffleVector, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertBefore) { + : Instruction( + VectorType::get(cast(V1->getType())->getElementType(), + cast(Mask->getType())->getElementCount()), + ShuffleVector, OperandTraits::op_begin(this), + OperandTraits::operands(this), InsertBefore) { assert(isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"); + Op<0>() = V1; Op<1>() = V2; - Op<2>() = Mask; + SmallVector MaskArr; + getShuffleMask(cast(Mask), MaskArr); + setShuffleMask(MaskArr); setName(Name); } ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask, - const Twine &Name, - BasicBlock *InsertAtEnd) -: Instruction(VectorType::get(cast(V1->getType())->getElementType(), - cast(Mask->getType())->getElementCount()), - ShuffleVector, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertAtEnd) { + const Twine &Name, BasicBlock *InsertAtEnd) + : Instruction( + VectorType::get(cast(V1->getType())->getElementType(), + cast(Mask->getType())->getElementCount()), + ShuffleVector, OperandTraits::op_begin(this), + OperandTraits::operands(this), InsertAtEnd) { assert(isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"); Op<0>() = V1; Op<1>() = V2; - Op<2>() = Mask; + SmallVector MaskArr; + getShuffleMask(cast(Mask), MaskArr); + setShuffleMask(MaskArr); + setName(Name); +} + +ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef Mask, + const Twine &Name, + Instruction *InsertBefore) + : Instruction( + VectorType::get(cast(V1->getType())->getElementType(), + Mask.size(), V1->getType()->getVectorIsScalable()), + ShuffleVector, OperandTraits::op_begin(this), + OperandTraits::operands(this), InsertBefore) { + assert(isValidOperands(V1, V2, Mask) && + "Invalid shuffle vector instruction operands!"); + Op<0>() = V1; + Op<1>() = V2; + setShuffleMask(Mask); + setName(Name); +} + +ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef Mask, + const Twine &Name, BasicBlock *InsertAtEnd) + : Instruction( + VectorType::get(cast(V1->getType())->getElementType(), + Mask.size(), V1->getType()->getVectorIsScalable()), + ShuffleVector, OperandTraits::op_begin(this), + OperandTraits::operands(this), InsertAtEnd) { + assert(isValidOperands(V1, V2, Mask) && + "Invalid shuffle vector instruction operands!"); + + Op<0>() = V1; + Op<1>() = V2; + setShuffleMask(Mask); setName(Name); } void ShuffleVectorInst::commute() { int NumOpElts = Op<0>()->getType()->getVectorNumElements(); - int NumMaskElts = getMask()->getType()->getVectorNumElements(); - SmallVector NewMask(NumMaskElts); - Type *Int32Ty = Type::getInt32Ty(getContext()); + int NumMaskElts = ShuffleMask.size(); + SmallVector NewMask(NumMaskElts); for (int i = 0; i != NumMaskElts; ++i) { int MaskElt = getMaskValue(i); - if (MaskElt == -1) { - NewMask[i] = UndefValue::get(Int32Ty); + if (MaskElt == UndefMaskElem) { + NewMask[i] = UndefMaskElem; continue; } assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask"); MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts; - NewMask[i] = ConstantInt::get(Int32Ty, MaskElt); + NewMask[i] = MaskElt; } - Op<2>() = ConstantVector::get(NewMask); + setShuffleMask(NewMask); Op<0>().swap(Op<1>()); } +bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, + ArrayRef Mask) { + // V1 and V2 must be vectors of the same type. + if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType()) + return false; + + // Make sure the mask elements make sense. + int V1Size = cast(V1->getType())->getNumElements(); + for (int Elem : Mask) + if (Elem != UndefMaskElem && Elem >= V1Size * 2) + return false; + + if (V1->getType()->getVectorIsScalable()) + if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask)) + return false; + + return true; +} + bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, const Value *Mask) { // V1 and V2 must be vectors of the same type. @@ -1902,7 +1954,8 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, // Mask must be vector of i32. auto *MaskTy = dyn_cast(Mask->getType()); - if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32)) + if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) || + MaskTy->isScalable() != V1->getType()->getVectorIsScalable()) return false; // Check to see if Mask is valid. @@ -1930,34 +1983,12 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, return true; } - // The bitcode reader can create a place holder for a forward reference - // used as the shuffle mask. When this occurs, the shuffle mask will - // fall into this case and fail. To avoid this error, do this bit of - // ugliness to allow such a mask pass. - if (const auto *CE = dyn_cast(Mask)) - if (CE->getOpcode() == Instruction::UserOp1) - return true; - return false; } -int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) { - assert(i < Mask->getType()->getVectorNumElements() && "Index out of range"); - assert(!Mask->getType()->getVectorElementCount().Scalable && - "Length of scalable vectors unknown at compile time"); - if (auto *CDS = dyn_cast(Mask)) - return CDS->getElementAsInteger(i); - Constant *C = Mask->getAggregateElement(i); - if (isa(C)) - return -1; - return cast(C)->getZExtValue(); -} - void ShuffleVectorInst::getShuffleMask(const Constant *Mask, SmallVectorImpl &Result) { - assert(!Mask->getType()->getVectorElementCount().Scalable && - "Length of scalable vectors unknown at compile time"); - unsigned NumElts = Mask->getType()->getVectorNumElements(); + unsigned NumElts = Mask->getType()->getVectorElementCount().Min; if (isa(Mask)) { Result.resize(NumElts, 0); return; @@ -1975,6 +2006,30 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask, } } +void ShuffleVectorInst::setShuffleMask(ArrayRef Mask) { + ShuffleMask.assign(Mask.begin(), Mask.end()); + ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType()); +} +Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef Mask, + Type *ResultTy) { + Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext()); + if (ResultTy->getVectorIsScalable()) { + assert(is_splat(Mask) && "Unexpected shuffle"); + Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true); + if (Mask[0] == 0) + return Constant::getNullValue(VecTy); + return UndefValue::get(VecTy); + } + SmallVector MaskConst; + for (int Elem : Mask) { + if (Elem == UndefMaskElem) + MaskConst.push_back(UndefValue::get(Int32Ty)); + else + MaskConst.push_back(ConstantInt::get(Int32Ty, Elem)); + } + return ConstantVector::get(MaskConst); +} + static bool isSingleSourceMaskImpl(ArrayRef Mask, int NumOpElts) { assert(!Mask.empty() && "Shuffle mask must contain elements"); bool UsesLHS = false; @@ -2124,7 +2179,7 @@ bool ShuffleVectorInst::isIdentityWithPadding() const { return false; // The first part of the mask must choose elements from exactly 1 source op. - SmallVector Mask = getShuffleMask(); + ArrayRef Mask = getShuffleMask(); if (!isIdentityMaskImpl(Mask, NumOpElts)) return false; @@ -4298,7 +4353,7 @@ InsertElementInst *InsertElementInst::cloneImpl() const { } ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const { - return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2)); + return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask()); } PHINode *PHINode::cloneImpl() const { return new PHINode(*this); } diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index 84364dbb230..a31a91766eb 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -3310,7 +3310,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) { void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) { Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1), - SV.getOperand(2)), + SV.getShuffleMask()), "Invalid shufflevector operands!", &SV); visitInstruction(SV); } diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index 149de47e3df..523ffe993ef 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9207,10 +9207,10 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) { return FullVT->getNumElements() == 2 * HalfVT->getNumElements(); }; - Constant *M1, *M2; + ArrayRef M1, M2; Value *S1Op1, *S2Op1; - if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) || - !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2)))) + if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) || + !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2)))) return false; // Check that the operands are half as wide as the result and we extract diff --git a/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp index 395c04161ea..b7baecdec6b 100644 --- a/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp +++ b/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp @@ -210,7 +210,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) { Arg.replaceAllUsesWith(NewVal); } else if (IsV3) { Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty), - {0, 1, 2}, + ArrayRef{0, 1, 2}, Arg.getName() + ".load"); Arg.replaceAllUsesWith(Shuf); } else { diff --git a/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp index cf641e3948a..5a2cd48a81e 100644 --- a/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp +++ b/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp @@ -409,7 +409,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) { DL->getTypeSizeInBits(Val->getType())) { assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType())); Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()), - { 0, 1, 2 }); + ArrayRef{0, 1, 2}); } Val = B.CreateBitCast(Val, EffectiveEltTy); diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index a2de0adb34d..6d03126b2db 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -15547,7 +15547,7 @@ bool ARMTargetLowering::shouldSinkOperands(Instruction *I, return false; if (!match(I->getOperand(Op), m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()), - m_Undef(), m_Zero()))) { + m_Undef(), m_ZeroMask()))) { return false; } Instruction *Shuffle = cast(I->getOperand(Op)); diff --git a/lib/Target/ARM/MVETailPredication.cpp b/lib/Target/ARM/MVETailPredication.cpp index dd7867397b4..26b71ba5b60 100644 --- a/lib/Target/ARM/MVETailPredication.cpp +++ b/lib/Target/ARM/MVETailPredication.cpp @@ -286,7 +286,7 @@ bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) { Instruction *Insert = nullptr; // The shuffle which broadcasts the index iv into a vector. if (!match(BroadcastSplat, - m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero()))) + m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask()))) return false; // The insert element which initialises a vector with the index iv. @@ -409,7 +409,7 @@ static bool MatchElemCountLoopSetup(Loop *L, Instruction *Shuffle, Instruction *Insert = nullptr; if (!match(Shuffle, - m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero()))) + m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask()))) return false; // Insert the limit into a vector. diff --git a/lib/Target/X86/X86PartialReduction.cpp b/lib/Target/X86/X86PartialReduction.cpp index daa293d1e74..26c05eab45b 100644 --- a/lib/Target/X86/X86PartialReduction.cpp +++ b/lib/Target/X86/X86PartialReduction.cpp @@ -384,7 +384,7 @@ bool X86PartialReduction::trySADReplacement(Value *Op, BinaryOperator *Add) { NumElts = OpTy->getVectorNumElements(); if (NumElts == 2) { // Extract down to 2 elements. - Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], {0, 1}); + Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef{0, 1}); } else if (NumElts >= 8) { SmallVector ConcatMask(NumElts); unsigned SubElts = Ops[0]->getType()->getVectorNumElements(); diff --git a/lib/Transforms/InstCombine/InstCombineCasts.cpp b/lib/Transforms/InstCombine/InstCombineCasts.cpp index 12fb8e9e96f..6b1824048f7 100644 --- a/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -642,12 +642,12 @@ static Instruction *shrinkSplatShuffle(TruncInst &Trunc, InstCombiner::BuilderTy &Builder) { auto *Shuf = dyn_cast(Trunc.getOperand(0)); if (Shuf && Shuf->hasOneUse() && isa(Shuf->getOperand(1)) && - Shuf->getMask()->getSplatValue() && + is_splat(Shuf->getShuffleMask()) && Shuf->getType() == Shuf->getOperand(0)->getType()) { // trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask Constant *NarrowUndef = UndefValue::get(Trunc.getType()); Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType()); - return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask()); + return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask()); } return nullptr; @@ -2553,7 +2553,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) { Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy); // Return a new shuffle vector. Use the same element ID's, as we // know the vector types match #elts. - return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2)); + return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask()); } } diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp index c8276c75918..e2c53ae0b31 100644 --- a/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -2814,11 +2814,10 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp, return nullptr; Value *Vec; - Constant *Mask; - if (match(BCSrcOp, - m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) { + ArrayRef Mask; + if (match(BCSrcOp, m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) { // Check whether every element of Mask is the same constant - if (auto *Elem = dyn_cast_or_null(Mask->getSplatValue())) { + if (is_splat(Mask)) { auto *VecTy = cast(BCSrcOp->getType()); auto *EltTy = cast(VecTy->getElementType()); if (C->isSplat(EltTy->getBitWidth())) { @@ -2827,6 +2826,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp, // then: // => %E = extractelement %vec, i32 Elem // icmp iK %SplatVal, + Value *Elem = Builder.getInt32(Mask[0]); Value *Extract = Builder.CreateExtractElement(Vec, Elem); Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth())); return new ICmpInst(Pred, Extract, NewC); @@ -5380,15 +5380,15 @@ static Instruction *foldVectorCmp(CmpInst &Cmp, bool IsFP = isa(Cmp); Value *V1, *V2; - Constant *M; - if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M)))) + ArrayRef M; + if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M)))) return nullptr; // If both arguments of the cmp are shuffles that use the same mask and // shuffle within a single vector, move the shuffle after the cmp: // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M Type *V1Ty = V1->getType(); - if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) && + if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) && V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) { Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2) : Builder.CreateICmp(Pred, V1, V2); @@ -5405,16 +5405,16 @@ static Instruction *foldVectorCmp(CmpInst &Cmp, // Length-changing splats are ok, so adjust the constants as needed: // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true); - Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true); - if (ScalarC && ScalarM) { + int MaskSplatIndex; + if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) { // We allow undefs in matching, but this transform removes those for safety. // Demanded elements analysis should be able to recover some/all of that. C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC); - M = ConstantVector::getSplat(M->getType()->getVectorElementCount(), - ScalarM); + SmallVector NewM(M.size(), MaskSplatIndex); Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C) : Builder.CreateICmp(Pred, V1, C); - return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M); + return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), + NewM); } return nullptr; diff --git a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 90b00536b47..6edf11ae665 100644 --- a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1389,7 +1389,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, Shuffle->getOperand(0)->getType()->getVectorNumElements(); // Handle trivial case of a splat. Only check the first element of LHS // operand. - if (isa(Shuffle->getMask()) && + if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) && DemandedElts.isAllOnesValue()) { if (!isa(I->getOperand(1))) { I->setOperand(1, UndefValue::get(I->getOperand(1)->getType())); @@ -1518,15 +1518,14 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, } if (NewUndefElts) { // Add additional discovered undefs. - SmallVector Elts; + SmallVector Elts; for (unsigned i = 0; i < VWidth; ++i) { if (UndefElts[i]) - Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext()))); + Elts.push_back(UndefMaskElem); else - Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()), - Shuffle->getMaskValue(i))); + Elts.push_back(Shuffle->getMaskValue(i)); } - I->setOperand(2, ConstantVector::get(Elts)); + Shuffle->setShuffleMask(Elts); MadeChange = true; } break; diff --git a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index b866f44779e..2d72696ee53 100644 --- a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -730,7 +730,7 @@ Instruction *InstCombiner::visitInsertValueInst(InsertValueInst &I) { } static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) { - int MaskSize = Shuf.getMask()->getType()->getVectorNumElements(); + int MaskSize = Shuf.getShuffleMask().size(); int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements(); // A vector select does not change the size of the operands. @@ -842,13 +842,10 @@ static Instruction *foldInsEltIntoSplat(InsertElementInst &InsElt) { // inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1 // --> shuf (inselt undef, X, 0), undef, <0,0,0,undef> unsigned NumMaskElts = Shuf->getType()->getVectorNumElements(); - SmallVector NewMaskVec(NumMaskElts); - Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext()); - Constant *Zero = ConstantInt::getNullValue(I32Ty); + SmallVector NewMask(NumMaskElts); for (unsigned i = 0; i != NumMaskElts; ++i) - NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i); + NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i); - Constant *NewMask = ConstantVector::get(NewMaskVec); return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask); } @@ -878,26 +875,23 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) { // For example: // inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask' unsigned NumMaskElts = Shuf->getType()->getVectorNumElements(); - SmallVector NewMaskVec(NumMaskElts); - Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext()); - Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC); - Constant *OldMask = Shuf->getMask(); + SmallVector NewMask(NumMaskElts); + ArrayRef OldMask = Shuf->getShuffleMask(); for (unsigned i = 0; i != NumMaskElts; ++i) { if (i != IdxC) { // All mask elements besides the inserted element remain the same. - NewMaskVec[i] = OldMask->getAggregateElement(i); - } else if (OldMask->getAggregateElement(i) == NewMaskEltC) { + NewMask[i] = OldMask[i]; + } else if (OldMask[i] == (int)IdxC) { // If the mask element was already set, there's nothing to do // (demanded elements analysis may unset it later). return nullptr; } else { - assert(isa(OldMask->getAggregateElement(i)) && + assert(OldMask[i] == UndefMaskElem && "Unexpected shuffle mask element for identity shuffle"); - NewMaskVec[i] = NewMaskEltC; + NewMask[i] = IdxC; } } - Constant *NewMask = ConstantVector::get(NewMaskVec); return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask); } @@ -965,27 +959,25 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) { // mask vector with the insertelt index plus the length of the vector // (because the constant vector operand of a shuffle is always the 2nd // operand). - Constant *Mask = Shuf->getMask(); - unsigned NumElts = Mask->getType()->getVectorNumElements(); + ArrayRef Mask = Shuf->getShuffleMask(); + unsigned NumElts = Mask.size(); SmallVector NewShufElts(NumElts); - SmallVector NewMaskElts(NumElts); + SmallVector NewMaskElts(NumElts); for (unsigned I = 0; I != NumElts; ++I) { if (I == InsEltIndex) { NewShufElts[I] = InsEltScalar; - Type *Int32Ty = Type::getInt32Ty(Shuf->getContext()); - NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts); + NewMaskElts[I] = InsEltIndex + NumElts; } else { // Copy over the existing values. NewShufElts[I] = ShufConstVec->getAggregateElement(I); - NewMaskElts[I] = Mask->getAggregateElement(I); + NewMaskElts[I] = Mask[I]; } } // Create new operands for a shuffle that includes the constant of the // original insertelt. The old shuffle will be dead now. return new ShuffleVectorInst(Shuf->getOperand(0), - ConstantVector::get(NewShufElts), - ConstantVector::get(NewMaskElts)); + ConstantVector::get(NewShufElts), NewMaskElts); } else if (auto *IEI = dyn_cast(Inst)) { // Transform sequences of insertelements ops with constant data/indexes into // a single shuffle op. @@ -1305,17 +1297,9 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef Mask) { if (isa(V)) return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size())); - if (Constant *C = dyn_cast(V)) { - SmallVector MaskValues; - for (int i = 0, e = Mask.size(); i != e; ++i) { - if (Mask[i] == -1) - MaskValues.push_back(UndefValue::get(I32Ty)); - else - MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i])); - } + if (Constant *C = dyn_cast(V)) return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()), - ConstantVector::get(MaskValues)); - } + Mask); Instruction *I = cast(V); switch (I->getOpcode()) { @@ -1404,7 +1388,7 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef Mask) { // Shuffles to: |EE|FF|GG|HH| // +--+--+--+--+ static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI, - SmallVector &Mask) { + ArrayRef Mask) { unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements(); unsigned MaskElems = Mask.size(); unsigned BegIdx = Mask.front(); @@ -1487,12 +1471,12 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { // Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4} // Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4} // The existing binop constant vector remains in the same operand position. - Constant *Mask = Shuf.getMask(); + ArrayRef Mask = Shuf.getShuffleMask(); Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) : ConstantExpr::getShuffleVector(IdC, C, Mask); bool MightCreatePoisonOrUB = - Mask->containsUndefElement() && + is_contained(Mask, UndefMaskElem) && (Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode)); if (MightCreatePoisonOrUB) NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true); @@ -1506,7 +1490,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { // An undef shuffle mask element may propagate as an undef constant element in // the new binop. That would produce poison where the original code might not. // If we already made a safe constant, then there's no danger. - if (Mask->containsUndefElement() && !MightCreatePoisonOrUB) + if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB) NewBO->dropPoisonGeneratingFlags(); return NewBO; } @@ -1518,14 +1502,14 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder) { Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1); - Constant *Mask = Shuf.getMask(); + ArrayRef Mask = Shuf.getShuffleMask(); Value *X; uint64_t IndexC; // Match a shuffle that is a splat to a non-zero element. if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X), m_ConstantInt(IndexC)))) || - !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0) + !match(Op1, m_Undef()) || match(Mask, m_ZeroMask()) || IndexC == 0) return nullptr; // Insert into element 0 of an undef vector. @@ -1538,12 +1522,12 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf, // shuf (inselt undef, X, 2), undef, <2,2,undef> // --> shuf (inselt undef, X, 0), undef, <0,0,undef> unsigned NumMaskElts = Shuf.getType()->getVectorNumElements(); - SmallVector NewMask(NumMaskElts, Zero); + SmallVector NewMask(NumMaskElts, 0); for (unsigned i = 0; i != NumMaskElts; ++i) - if (isa(Mask->getAggregateElement(i))) - NewMask[i] = Mask->getAggregateElement(i); + if (Mask[i] == UndefMaskElem) + NewMask[i] = Mask[i]; - return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask)); + return new ShuffleVectorInst(NewIns, UndefVec, NewMask); } /// Try to fold shuffles that are the equivalent of a vector select. @@ -1612,14 +1596,14 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf, BinaryOperator::BinaryOps BOpc = Opc0; // Select the constant elements needed for the single binop. - Constant *Mask = Shuf.getMask(); + ArrayRef Mask = Shuf.getShuffleMask(); Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask); // We are moving a binop after a shuffle. When a shuffle has an undefined // mask element, the result is undefined, but it is not poison or undefined // behavior. That is not necessarily true for div/rem/shift. bool MightCreatePoisonOrUB = - Mask->containsUndefElement() && + is_contained(Mask, UndefMaskElem) && (Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc)); if (MightCreatePoisonOrUB) NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1); @@ -1668,7 +1652,7 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf, NewBO->andIRFlags(B1); if (DropNSW) NewBO->setHasNoSignedWrap(false); - if (Mask->containsUndefElement() && !MightCreatePoisonOrUB) + if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB) NewBO->dropPoisonGeneratingFlags(); return NewBO; } @@ -1694,8 +1678,7 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf, // and have the same number of elements as this shuffle. unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements(); Value *NarrowCond; - if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(), - m_Constant()))) || + if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) || NarrowCond->getType()->getVectorNumElements() != NarrowNumElts || !cast(Cond)->isIdentityWithPadding()) return nullptr; @@ -1703,8 +1686,8 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf, // shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) --> // sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask) Value *Undef = UndefValue::get(X->getType()); - Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask()); - Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask()); + Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask()); + Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask()); return SelectInst::Create(NarrowCond, NarrowX, NarrowY); } @@ -1715,8 +1698,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) { return nullptr; Value *X, *Y; - Constant *Mask; - if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask)))) + ArrayRef Mask; + if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Mask(Mask)))) return nullptr; // Be conservative with shuffle transforms. If we can't kill the 1st shuffle, @@ -1736,16 +1719,16 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) { // shuf (shuf X, Y, ), undef, <0, undef, 2, 3> --> // shuf X, Y, unsigned NumElts = Shuf.getType()->getVectorNumElements(); - SmallVector NewMask(NumElts); - assert(NumElts < Mask->getType()->getVectorNumElements() && + SmallVector NewMask(NumElts); + assert(NumElts < Mask.size() && "Identity with extract must have less elements than its inputs"); for (unsigned i = 0; i != NumElts; ++i) { - Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i); - Constant *MaskElt = Mask->getAggregateElement(i); - NewMask[i] = isa(ExtractMaskElt) ? ExtractMaskElt : MaskElt; + int ExtractMaskElt = Shuf.getMaskValue(i); + int MaskElt = Mask[i]; + NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt; } - return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask)); + return new ShuffleVectorInst(X, Y, NewMask); } /// Try to replace a shuffle with an insertelement or try to replace a shuffle @@ -1753,7 +1736,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) { static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf, InstCombiner &IC) { Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1); - SmallVector Mask = Shuf.getShuffleMask(); + SmallVector Mask; + Shuf.getShuffleMask(Mask); // The shuffle must not change vector sizes. // TODO: This restriction could be removed if the insert has only one use @@ -1872,7 +1856,7 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) { assert(WideElts > NarrowElts && "Unexpected types for identity with padding"); Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext()); - SmallVector Mask = Shuf.getShuffleMask(); + ArrayRef Mask = Shuf.getShuffleMask(); SmallVector NewMask(Mask.size(), UndefValue::get(I32Ty)); for (int i = 0, e = Mask.size(); i != e; ++i) { if (Mask[i] == -1) @@ -1905,28 +1889,28 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) { Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) { Value *LHS = SVI.getOperand(0); Value *RHS = SVI.getOperand(1); - if (auto *V = SimplifyShuffleVectorInst( - LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI))) + if (auto *V = + SimplifyShuffleVectorInst(LHS, RHS, SVI.getShuffleMask(), + SVI.getType(), SQ.getWithInstruction(&SVI))) return replaceInstUsesWith(SVI, V); // shuffle x, x, mask --> shuffle x, undef, mask' unsigned VWidth = SVI.getType()->getVectorNumElements(); unsigned LHSWidth = LHS->getType()->getVectorNumElements(); - SmallVector Mask = SVI.getShuffleMask(); + ArrayRef Mask = SVI.getShuffleMask(); Type *Int32Ty = Type::getInt32Ty(SVI.getContext()); if (LHS == RHS) { assert(!isa(RHS) && "Shuffle with 2 undef ops not simplified?"); // Remap any references to RHS to use LHS. - SmallVector Elts; + SmallVector Elts; for (unsigned i = 0; i != VWidth; ++i) { // Propagate undef elements or force mask to LHS. if (Mask[i] < 0) - Elts.push_back(UndefValue::get(Int32Ty)); + Elts.push_back(UndefMaskElem); else - Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth)); + Elts.push_back(Mask[i] % LHSWidth); } - return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), - ConstantVector::get(Elts)); + return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), Elts); } // shuffle undef, x, mask --> shuffle x, undef, mask' @@ -2151,8 +2135,8 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) { if (newLHS == LHS && newRHS == RHS) return MadeChange ? &SVI : nullptr; - SmallVector LHSMask; - SmallVector RHSMask; + ArrayRef LHSMask; + ArrayRef RHSMask; if (newLHS != LHS) LHSMask = LHSShuffle->getShuffleMask(); if (RHSShuffle && newRHS != RHS) diff --git a/lib/Transforms/InstCombine/InstructionCombining.cpp b/lib/Transforms/InstCombine/InstructionCombining.cpp index 68797662f45..23370cf21f0 100644 --- a/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1534,9 +1534,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // narrow binop on each pair of the source operands followed by concatenation // of the results. Value *L0, *L1, *R0, *R1; - Constant *Mask; - if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) && - match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) && + ArrayRef Mask; + if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) && + match(RHS, + m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) && LHS->hasOneUse() && RHS->hasOneUse() && cast(LHS)->isConcat() && cast(RHS)->isConcat()) { @@ -1560,7 +1561,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { if (!isSafeToSpeculativelyExecute(&Inst)) return nullptr; - auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) { + auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef M) { Value *XY = Builder.CreateBinOp(Opcode, X, Y); if (auto *BO = dyn_cast(XY)) BO->copyIRFlags(&Inst); @@ -1570,8 +1571,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // If both arguments of the binary operation are shuffles that use the same // mask and shuffle within a single vector, move the shuffle after the binop. Value *V1, *V2; - if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) && - match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) && + if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) && + match(RHS, + m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) && V1->getType() == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) { // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask) @@ -1581,17 +1583,19 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // If both arguments of a commutative binop are select-shuffles that use the // same mask with commuted operands, the shuffles are unnecessary. if (Inst.isCommutative() && - match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) && + match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) && match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1), - m_Specific(Mask)))) { + m_SpecificMask(Mask)))) { auto *LShuf = cast(LHS); auto *RShuf = cast(RHS); // TODO: Allow shuffles that contain undefs in the mask? // That is legal, but it reduces undef knowledge. // TODO: Allow arbitrary shuffles by shuffling after binop? // That might be legal, but we have to deal with poison. - if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() && - RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) { + if (LShuf->isSelect() && + !is_contained(LShuf->getShuffleMask(), UndefMaskElem) && + RShuf->isSelect() && + !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) { // Example: // LHS = shuffle V1, V2, <0, 5, 6, 3> // RHS = shuffle V2, V1, <0, 5, 6, 3> @@ -1608,9 +1612,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // other binops, so they can be folded. It may also enable demanded elements // transforms. Constant *C; - if (match(&Inst, m_c_BinOp( - m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))), - m_Constant(C))) && + if (match(&Inst, m_c_BinOp(m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), + m_Mask(Mask))), + m_Constant(C))) && V1->getType()->getVectorNumElements() <= NumElts) { assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() && "Shuffle should not change scalar type"); @@ -1621,8 +1625,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // reorder is not possible. A 1-to-1 mapping is not required. Example: // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = bool ConstOp1 = isa(RHS); - SmallVector ShMask; - ShuffleVectorInst::getShuffleMask(Mask, ShMask); + ArrayRef ShMask = Mask; unsigned SrcVecNumElts = V1->getType()->getVectorNumElements(); UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType()); SmallVector NewVecC(SrcVecNumElts, UndefScalar); @@ -1687,12 +1690,12 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { std::swap(LHS, RHS); Value *X; - Constant *MaskC; - const APInt *SplatIndex; + ArrayRef MaskC; + int SplatIndex; BinaryOperator *BO; if (!match(LHS, m_OneUse(m_ShuffleVector(m_Value(X), m_Undef(), - m_Constant(MaskC)))) || - !match(MaskC, m_APIntAllowUndef(SplatIndex)) || + m_Mask(MaskC)))) || + !match(MaskC, m_SplatOrUndefMask(SplatIndex)) || X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) || BO->getOpcode() != Opcode) return nullptr; @@ -1701,10 +1704,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // moving 'Y' before the splat shuffle, we are implicitly assuming // that it is not undef/poison at the splat index. Value *Y, *OtherOp; - if (isSplatValue(BO->getOperand(0), SplatIndex->getZExtValue())) { + if (isSplatValue(BO->getOperand(0), SplatIndex)) { Y = BO->getOperand(0); OtherOp = BO->getOperand(1); - } else if (isSplatValue(BO->getOperand(1), SplatIndex->getZExtValue())) { + } else if (isSplatValue(BO->getOperand(1), SplatIndex)) { Y = BO->getOperand(1); OtherOp = BO->getOperand(0); } else { @@ -1716,7 +1719,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) { // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp Value *NewBO = Builder.CreateBinOp(Opcode, X, Y); UndefValue *Undef = UndefValue::get(Inst.getType()); - Constant *NewMask = ConstantInt::get(MaskC->getType(), *SplatIndex); + SmallVector NewMask(MaskC.size(), SplatIndex); Value *NewSplat = Builder.CreateShuffleVector(NewBO, Undef, NewMask); Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp); diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp index dd892921b62..ea6156f4c56 100644 --- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -1867,10 +1867,9 @@ struct MemorySanitizerVisitor : public InstVisitor { } void visitShuffleVectorInst(ShuffleVectorInst &I) { - insertShadowCheck(I.getOperand(2), &I); IRBuilder<> IRB(&I); setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1), - I.getOperand(2), "_msprop")); + I.getShuffleMask(), "_msprop")); setOriginForNaryOp(I); } diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp index 6b9792aa77b..d831b0da37a 100644 --- a/lib/Transforms/Scalar/GVN.cpp +++ b/lib/Transforms/Scalar/GVN.cpp @@ -289,7 +289,7 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) { e.commutative = true; } - if (CmpInst *C = dyn_cast(I)) { + if (auto *C = dyn_cast(I)) { // Sort the operand value numbers so xx get the same value number. CmpInst::Predicate Predicate = C->getPredicate(); if (e.varargs[0] > e.varargs[1]) { @@ -298,10 +298,11 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) { } e.opcode = (C->getOpcode() << 8) | Predicate; e.commutative = true; - } else if (InsertValueInst *E = dyn_cast(I)) { - for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end(); - II != IE; ++II) - e.varargs.push_back(*II); + } else if (auto *E = dyn_cast(I)) { + e.varargs.append(E->idx_begin(), E->idx_end()); + } else if (auto *SVI = dyn_cast(I)) { + ArrayRef ShuffleMask = SVI->getShuffleMask(); + e.varargs.append(ShuffleMask.begin(), ShuffleMask.end()); } return e; @@ -1732,7 +1733,8 @@ uint32_t GVN::ValueTable::phiTranslateImpl(const BasicBlock *Pred, // instead of value numbers. Those index numbers should not be // translated. if ((i > 1 && Exp.opcode == Instruction::InsertValue) || - (i > 0 && Exp.opcode == Instruction::ExtractValue)) + (i > 0 && Exp.opcode == Instruction::ExtractValue) || + (i > 1 && Exp.opcode == Instruction::ShuffleVector)) continue; Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn); } diff --git a/lib/Transforms/Scalar/GVNSink.cpp b/lib/Transforms/Scalar/GVNSink.cpp index 6b9a88d04ed..d0b96218137 100644 --- a/lib/Transforms/Scalar/GVNSink.cpp +++ b/lib/Transforms/Scalar/GVNSink.cpp @@ -350,6 +350,7 @@ using ModelledPHISet = DenseSet>; class InstructionUseExpr : public GVNExpression::BasicExpression { unsigned MemoryUseOrder = -1; bool Volatile = false; + std::vector ShuffleMask; public: InstructionUseExpr(Instruction *I, ArrayRecycler &R, @@ -366,15 +367,18 @@ public: void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; } void setVolatile(bool V) { Volatile = V; } + void setShuffleMask(ArrayRef Mask) { + ShuffleMask.assign(Mask.begin(), Mask.end()); + } hash_code getHashValue() const override { return hash_combine(GVNExpression::BasicExpression::getHashValue(), - MemoryUseOrder, Volatile); + MemoryUseOrder, Volatile, ArrayRef(ShuffleMask)); } template hash_code getHashValue(Function MapFn) { - hash_code H = - hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile); + hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile, + ArrayRef(ShuffleMask)); for (auto *V : operands()) H = hash_combine(H, MapFn(V)); return H; @@ -402,6 +406,8 @@ class ValueTable { CmpInst::Predicate Predicate = C->getPredicate(); E->setOpcode((C->getOpcode() << 8) | Predicate); } + if (ShuffleVectorInst *SVI = dyn_cast(I)) + E->setShuffleMask(SVI->getShuffleMask()); return E; } diff --git a/lib/Transforms/Scalar/NewGVN.cpp b/lib/Transforms/Scalar/NewGVN.cpp index 6a643480f31..17b2c534513 100644 --- a/lib/Transforms/Scalar/NewGVN.cpp +++ b/lib/Transforms/Scalar/NewGVN.cpp @@ -2030,10 +2030,12 @@ NewGVN::performSymbolicEvaluation(Value *V, case Instruction::Select: case Instruction::ExtractElement: case Instruction::InsertElement: - case Instruction::ShuffleVector: case Instruction::GetElementPtr: E = createExpression(I); break; + case Instruction::ShuffleVector: + // FIXME: Add support for shufflevector to createExpression. + return nullptr; default: return nullptr; } diff --git a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index fe344ce5f8d..b8b98d67333 100644 --- a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1009,7 +1009,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { auto *SV = cast(I); UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType()); std::string Name = suffixed_name_or(I, ".base", "base_sv"); - return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2), + return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(), Name, SV); } }; diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index 3a89fd82938..af51dea002d 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3405,7 +3405,7 @@ void InnerLoopVectorizer::truncateToMinimalBitwidths() { auto *O1 = B.CreateZExtOrTrunc( SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1)); - NewI = B.CreateShuffleVector(O0, O1, SI->getMask()); + NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask()); } else if (isa(I) || isa(I)) { // Don't do anything with the operands, just extend the result. continue; diff --git a/unittests/IR/PatternMatch.cpp b/unittests/IR/PatternMatch.cpp index cb11d1ec71f..186746bca79 100644 --- a/unittests/IR/PatternMatch.cpp +++ b/unittests/IR/PatternMatch.cpp @@ -991,31 +991,32 @@ TEST_F(PatternMatchTest, VectorOps) { EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt()))); // Test matching shufflevector - EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero()))); - EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C)))); + ArrayRef Mask; + EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask()))); + EXPECT_TRUE( + match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask)))); EXPECT_TRUE(A == VI3); EXPECT_TRUE(B == VI4); - EXPECT_TRUE(C == IdxVec); A = B = C = nullptr; // reset // Test matching the vector splat pattern EXPECT_TRUE(match( SI1, m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()), - m_Undef(), m_Zero()))); + m_Undef(), m_ZeroMask()))); EXPECT_FALSE(match( SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()), - m_Undef(), m_Zero()))); + m_Undef(), m_ZeroMask()))); EXPECT_FALSE(match( SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()), - m_Undef(), m_Zero()))); + m_Undef(), m_ZeroMask()))); EXPECT_TRUE(match( SP1, m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()), - m_Undef(), m_Zero()))); + m_Undef(), m_ZeroMask()))); EXPECT_TRUE(match( SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()), - m_Undef(), m_Zero()))); + m_Undef(), m_ZeroMask()))); EXPECT_TRUE(A == Val); }