From 2360e51fd085fdf34d720abf8cf2009bc9f35242 Mon Sep 17 00:00:00 2001 From: Andrea Di Biagio Date: Thu, 8 May 2014 17:44:04 +0000 Subject: [PATCH] [X86] Add target specific combine rules to fold SSE2/AVX2 packed arithmetic shift intrinsics. This patch teaches the backend how to combine packed SSE2/AVX2 arithmetic shift intrinsics. The rules are: - Always fold a packed arithmetic shift by zero to its first operand; - Convert a packed arithmetic shift intrinsic dag node into a ISD::SRA only if the shift count is known to be smaller than the vector element size. This patch also teaches to function 'getTargetVShiftByConstNode' how fold target specific vector shifts by zero. Added two new tests to verify that the DAGCombiner is able to fold sequences of SSE2/AVX2 packed arithmetic shift calls. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@208342 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/X86/X86ISelLowering.cpp | 55 +++++++++++++++++++++ test/CodeGen/X86/combine-avx2-intrinsics.ll | 51 +++++++++++++++++++ test/CodeGen/X86/combine-sse2-intrinsics.ll | 53 ++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 test/CodeGen/X86/combine-avx2-intrinsics.ll create mode 100644 test/CodeGen/X86/combine-sse2-intrinsics.ll diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 5355053241a..0ea98784eb3 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -1556,6 +1556,7 @@ void X86TargetLowering::resetOperationActions() { setTargetDAGCombine(ISD::TRUNCATE); setTargetDAGCombine(ISD::SINT_TO_FP); setTargetDAGCombine(ISD::SETCC); + setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); if (Subtarget->is64Bit()) setTargetDAGCombine(ISD::MUL); setTargetDAGCombine(ISD::XOR); @@ -11615,6 +11616,10 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT, SelectionDAG &DAG) { MVT ElementType = VT.getVectorElementType(); + // Fold this packed shift into its first operand if ShiftAmt is 0. + if (ShiftAmt == 0) + return SrcOp; + // Check for ShiftAmt >= element width if (ShiftAmt >= ElementType.getSizeInBits()) { if (Opc == X86ISD::VSRAI) @@ -18484,6 +18489,55 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG, return SDValue(); } +static SDValue PerformINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG) { + unsigned IntNo = cast(N->getOperand(0))->getZExtValue(); + switch (IntNo) { + default: return SDValue(); + // Packed SSE2/AVX2 arithmetic shift immediate intrinsics. + case Intrinsic::x86_sse2_psrai_w: + case Intrinsic::x86_sse2_psrai_d: + case Intrinsic::x86_avx2_psrai_w: + case Intrinsic::x86_avx2_psrai_d: + case Intrinsic::x86_sse2_psra_w: + case Intrinsic::x86_sse2_psra_d: + case Intrinsic::x86_avx2_psra_w: + case Intrinsic::x86_avx2_psra_d: { + SDValue Op0 = N->getOperand(1); + SDValue Op1 = N->getOperand(2); + EVT VT = Op0.getValueType(); + assert(VT.isVector() && "Expected a vector type!"); + + if (isa(Op1)) + Op1 = Op1.getOperand(0); + + if (!isa(Op1)) + return SDValue(); + + EVT SVT = VT.getVectorElementType(); + unsigned SVTBits = SVT.getSizeInBits(); + + ConstantSDNode *CND = cast(Op1); + const APInt &C = APInt(SVTBits, CND->getAPIntValue().getZExtValue()); + uint64_t ShAmt = C.getZExtValue(); + + // Don't try to convert this shift into a ISD::SRA if the shift + // count is bigger than or equal to the element size. + if (ShAmt >= SVTBits) + return SDValue(); + + // Trivial case: if the shift count is zero, then fold this + // into the first operand. + if (ShAmt == 0) + return Op0; + + // Replace this packed shift intrinsic with a target independent + // shift dag node. + SDValue Splat = DAG.getConstant(C, VT); + return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat); + } + } +} + /// PerformMulCombine - Optimize a single multiply with constant into two /// in order to implement it with two cheaper instructions, e.g. /// LEA + SHL, LEA + LEA. @@ -20304,6 +20358,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case X86ISD::VPERM2X128: case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget); case ISD::FMA: return PerformFMACombine(N, DAG, Subtarget); + case ISD::INTRINSIC_WO_CHAIN: return PerformINTRINSIC_WO_CHAINCombine(N, DAG); } return SDValue(); diff --git a/test/CodeGen/X86/combine-avx2-intrinsics.ll b/test/CodeGen/X86/combine-avx2-intrinsics.ll new file mode 100644 index 00000000000..0560a8d6ae2 --- /dev/null +++ b/test/CodeGen/X86/combine-avx2-intrinsics.ll @@ -0,0 +1,51 @@ +; RUN: llc < %s -march=x86-64 -mcpu=core-avx2 | FileCheck %s + +; Verify that the backend correctly combines AVX2 builtin intrinsics. + + +define <8 x i32> @test_psra_1(<8 x i32> %A) { + %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 3) + %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> ) + %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 2) + ret <8 x i32> %3 +} +; CHECK-LABEL: test_psra_1 +; CHECK: vpsrad $8, %ymm0, %ymm0 +; CHECK-NEXT: ret + +define <16 x i16> @test_psra_2(<16 x i16> %A) { + %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 3) + %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> ) + %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 2) + ret <16 x i16> %3 +} +; CHECK-LABEL: test_psra_2 +; CHECK: vpsraw $8, %ymm0, %ymm0 +; CHECK-NEXT: ret + +define <16 x i16> @test_psra_3(<16 x i16> %A) { + %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 0) + %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> ) + %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 0) + ret <16 x i16> %3 +} +; CHECK-LABEL: test_psra_3 +; CHECK-NOT: vpsraw +; CHECK: ret + +define <8 x i32> @test_psra_4(<8 x i32> %A) { + %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 0) + %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> ) + %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 0) + ret <8 x i32> %3 +} +; CHECK-LABEL: test_psra_4 +; CHECK-NOT: vpsrad +; CHECK: ret + + +declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) +declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) +declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) +declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) + diff --git a/test/CodeGen/X86/combine-sse2-intrinsics.ll b/test/CodeGen/X86/combine-sse2-intrinsics.ll new file mode 100644 index 00000000000..fa500e5d8d6 --- /dev/null +++ b/test/CodeGen/X86/combine-sse2-intrinsics.ll @@ -0,0 +1,53 @@ +; RUN: llc < %s -march=x86 -mcpu=core2 | FileCheck %s +; RUN: llc < %s -march=x86-64 -mcpu=corei7 | FileCheck %s + +; Verify that the backend correctly combines SSE2 builtin intrinsics. + + +define <4 x i32> @test_psra_1(<4 x i32> %A) { + %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 3) + %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> ) + %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %2, i32 2) + ret <4 x i32> %3 +} +; CHECK-LABEL: test_psra_1 +; CHECK: psrad $8, %xmm0 +; CHECK-NEXT: ret + +define <8 x i16> @test_psra_2(<8 x i16> %A) { + %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 3) + %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> ) + %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 2) + ret <8 x i16> %3 +} +; CHECK-LABEL: test_psra_2 +; CHECK: psraw $8, %xmm0 +; CHECK-NEXT: ret + +define <4 x i32> @test_psra_3(<4 x i32> %A) { + %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 0) + %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> ) + %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %2, i32 0) + ret <4 x i32> %3 +} +; CHECK-LABEL: test_psra_3 +; CHECK-NOT: psrad +; CHECK: ret + + +define <8 x i16> @test_psra_4(<8 x i16> %A) { + %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 0) + %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> ) + %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 0) + ret <8 x i16> %3 +} +; CHECK-LABEL: test_psra_4 +; CHECK-NOT: psraw +; CHECK: ret + + +declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) +declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) +declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) +