mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-12-04 01:11:44 +00:00
LoopVectorizer - skip 'bitcast' between GEP and load.
Skipping 'bitcast' in this case allows to vectorize load: %arrayidx = getelementptr inbounds double*, double** %in, i64 %indvars.iv %tmp53 = bitcast double** %arrayidx to i64* %tmp54 = load i64, i64* %tmp53, align 8 Differential Revision http://reviews.llvm.org/D14112 llvm-svn: 251907
This commit is contained in:
Elena Demikhovsky
parent
5991145dd5
commit
45c8421c3f
@ -268,6 +268,32 @@ static Type* ToVectorTy(Type *Scalar, unsigned VF) {
|
||||
return VectorType::get(Scalar, VF);
|
||||
}
|
||||
|
||||
/// A helper function that returns GEP instruction and knows to skip a
|
||||
/// 'bitcast'. The 'bitcast' may be skipped if the source and the destination
|
||||
/// pointee types of the 'bitcast' have the same size.
|
||||
/// For example:
|
||||
/// bitcast double** %var to i64* - can be skipped
|
||||
/// bitcast double** %var to i8* - can not
|
||||
static GetElementPtrInst *getGEPInstruction(Value *Ptr) {
|
||||
|
||||
if (isa<GetElementPtrInst>(Ptr))
|
||||
return cast<GetElementPtrInst>(Ptr);
|
||||
|
||||
if (isa<BitCastInst>(Ptr) &&
|
||||
isa<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0))) {
|
||||
Type *BitcastTy = Ptr->getType();
|
||||
Type *GEPTy = cast<BitCastInst>(Ptr)->getSrcTy();
|
||||
if (!isa<PointerType>(BitcastTy) || !isa<PointerType>(GEPTy))
|
||||
return nullptr;
|
||||
Type *Pointee1Ty = cast<PointerType>(BitcastTy)->getPointerElementType();
|
||||
Type *Pointee2Ty = cast<PointerType>(GEPTy)->getPointerElementType();
|
||||
const DataLayout &DL = cast<BitCastInst>(Ptr)->getModule()->getDataLayout();
|
||||
if (DL.getTypeSizeInBits(Pointee1Ty) == DL.getTypeSizeInBits(Pointee2Ty))
|
||||
return cast<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0));
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/// InnerLoopVectorizer vectorizes loops which contain only one basic
|
||||
/// block to a specified vectorization factor (VF).
|
||||
/// This class performs the widening of scalars into vectors, or multiple
|
||||
@ -1983,7 +2009,7 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
|
||||
return II.getConsecutiveDirection();
|
||||
}
|
||||
|
||||
GetElementPtrInst *Gep = dyn_cast_or_null<GetElementPtrInst>(Ptr);
|
||||
GetElementPtrInst *Gep = getGEPInstruction(Ptr);
|
||||
if (!Gep)
|
||||
return 0;
|
||||
|
||||
@ -2377,7 +2403,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
|
||||
VectorParts &Entry = WidenMap.get(Instr);
|
||||
|
||||
// Handle consecutive loads/stores.
|
||||
GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
|
||||
GetElementPtrInst *Gep = getGEPInstruction(Ptr);
|
||||
if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
|
||||
setDebugLocFromInst(Builder, Gep);
|
||||
Value *PtrOperand = Gep->getPointerOperand();
|
||||
|
||||
@ -1,114 +1,113 @@
|
||||
; RUN: opt < %s -loop-vectorize -pass-remarks-analysis='loop-vectorize' -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
|
||||
|
||||
; Verify analysis remarks are generated when interleaving is not beneficial.
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:4:14: the cost-model indicates that vectorization is not beneficial
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:4:14: the cost-model indicates that interleaving is not beneficial and is explicitly disabled or interleave count is set to 1
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:11:14: the cost-model indicates that vectorization is not beneficial
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:11:14: the cost-model indicates that interleaving is not beneficial
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:5:17: the cost-model indicates that vectorization is not beneficial
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:5:17: the cost-model indicates that interleaving is not beneficial and is explicitly disabled or interleave count is set to 1
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:12:17: the cost-model indicates that vectorization is not beneficial
|
||||
; CHECK: remark: vectorization-remarks-profitable.c:12:17: the cost-model indicates that interleaving is not beneficial
|
||||
|
||||
; First loop.
|
||||
; #pragma clang loop interleave(disable) unroll(disable)
|
||||
; for(int i = 0; i < n; i++) {
|
||||
; out[i] = in[i];
|
||||
; out[i] = *in[i];
|
||||
; }
|
||||
|
||||
; Second loop.
|
||||
; #pragma clang loop unroll(disable)
|
||||
; for(int i = 0; i < n; i++) {
|
||||
; out[i] = in[i];
|
||||
; out[i] = *in[i];
|
||||
; }
|
||||
|
||||
; ModuleID = 'vectorization-remarks-profitable.ll'
|
||||
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
|
||||
target triple = "x86_64-apple-macosx10.10.0"
|
||||
|
||||
; Function Attrs: nounwind ssp uwtable
|
||||
define void @do_not_interleave(float* nocapture %out, float* nocapture readonly %in, i32 %n) #0 {
|
||||
; Function Attrs: nounwind uwtable
|
||||
define void @do_not_interleave(float** noalias nocapture readonly %in, float* noalias nocapture %out, i32 %size) #0 {
|
||||
entry:
|
||||
%cmp.7 = icmp sgt i32 %n, 0, !dbg !3
|
||||
br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !8
|
||||
%cmp.4 = icmp eq i32 %size, 0, !dbg !10
|
||||
br i1 %cmp.4, label %for.end, label %for.body.preheader, !dbg !11
|
||||
|
||||
for.body.preheader: ; preds = %entry
|
||||
br label %for.body, !dbg !9
|
||||
|
||||
for.cond.cleanup.loopexit: ; preds = %for.body
|
||||
br label %for.cond.cleanup, !dbg !10
|
||||
|
||||
for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
|
||||
ret void, !dbg !10
|
||||
br label %for.body, !dbg !12
|
||||
|
||||
for.body: ; preds = %for.body.preheader, %for.body
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
|
||||
%arrayidx = getelementptr inbounds float, float* %in, i64 %indvars.iv, !dbg !9
|
||||
%0 = bitcast float* %arrayidx to i32*, !dbg !9
|
||||
%1 = load i32, i32* %0, align 4, !dbg !9, !tbaa !11
|
||||
%arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !15
|
||||
%2 = bitcast float* %arrayidx2 to i32*, !dbg !16
|
||||
store i32 %1, i32* %2, align 4, !dbg !16, !tbaa !11
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !8
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !8
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !8
|
||||
br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !8, !llvm.loop !17
|
||||
%arrayidx = getelementptr inbounds float*, float** %in, i64 %indvars.iv, !dbg !12
|
||||
%0 = bitcast float** %arrayidx to i32**, !dbg !12
|
||||
%1 = load i32*, i32** %0, align 8, !dbg !12
|
||||
%2 = load i32, i32* %1, align 4, !dbg !13
|
||||
%arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !14
|
||||
%3 = bitcast float* %arrayidx2 to i32*, !dbg !15
|
||||
store i32 %2, i32* %3, align 4, !dbg !15
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !11
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !11
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %size, !dbg !11
|
||||
br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !11, !llvm.loop !16
|
||||
|
||||
for.end.loopexit: ; preds = %for.body
|
||||
br label %for.end, !dbg !19
|
||||
|
||||
for.end: ; preds = %for.end.loopexit, %entry
|
||||
ret void, !dbg !19
|
||||
}
|
||||
|
||||
; Function Attrs: nounwind ssp uwtable
|
||||
define void @interleave_not_profitable(float* nocapture %out, float* nocapture readonly %in, i32 %n) #0 {
|
||||
; Function Attrs: nounwind uwtable
|
||||
define void @interleave_not_profitable(float** noalias nocapture readonly %in, float* noalias nocapture %out, i32 %size) #0 {
|
||||
entry:
|
||||
%cmp.7 = icmp sgt i32 %n, 0, !dbg !20
|
||||
br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !22
|
||||
%cmp.4 = icmp eq i32 %size, 0, !dbg !20
|
||||
br i1 %cmp.4, label %for.end, label %for.body, !dbg !21
|
||||
|
||||
for.body.preheader: ; preds = %entry
|
||||
br label %for.body, !dbg !23
|
||||
for.body: ; preds = %entry, %for.body
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
||||
%arrayidx = getelementptr inbounds float*, float** %in, i64 %indvars.iv, !dbg !22
|
||||
%0 = bitcast float** %arrayidx to i32**, !dbg !22
|
||||
%1 = load i32*, i32** %0, align 8, !dbg !22
|
||||
%2 = load i32, i32* %1, align 4, !dbg !23
|
||||
%arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !24
|
||||
%3 = bitcast float* %arrayidx2 to i32*, !dbg !25
|
||||
store i32 %2, i32* %3, align 4, !dbg !25
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %size, !dbg !21
|
||||
br i1 %exitcond, label %for.end, label %for.body, !dbg !21, !llvm.loop !26
|
||||
|
||||
for.cond.cleanup.loopexit: ; preds = %for.body
|
||||
br label %for.cond.cleanup, !dbg !24
|
||||
|
||||
for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry
|
||||
ret void, !dbg !24
|
||||
|
||||
for.body: ; preds = %for.body.preheader, %for.body
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
|
||||
%arrayidx = getelementptr inbounds float, float* %in, i64 %indvars.iv, !dbg !23
|
||||
%0 = bitcast float* %arrayidx to i32*, !dbg !23
|
||||
%1 = load i32, i32* %0, align 4, !dbg !23, !tbaa !11
|
||||
%arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !25
|
||||
%2 = bitcast float* %arrayidx2 to i32*, !dbg !26
|
||||
store i32 %1, i32* %2, align 4, !dbg !26, !tbaa !11
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !22
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !22
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !22
|
||||
br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !22, !llvm.loop !27
|
||||
for.end: ; preds = %for.body, %entry
|
||||
ret void, !dbg !27
|
||||
}
|
||||
|
||||
attributes #0 = { nounwind }
|
||||
attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
|
||||
|
||||
!llvm.module.flags = !{!0, !1}
|
||||
!llvm.ident = !{!2}
|
||||
!llvm.dbg.cu = !{!0}
|
||||
!llvm.module.flags = !{!7, !8}
|
||||
!llvm.ident = !{!9}
|
||||
|
||||
!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 250016)", isOptimized: false, runtimeVersion: 0, emissionKind: 2, enums: !2, subprograms: !3)
|
||||
!1 = !DIFile(filename: "vectorization-remarks-profitable.c", directory: "")
|
||||
!2 = !{}
|
||||
!3 = !{!4, !6}
|
||||
!4 = distinct !DISubprogram(name: "do_not_interleave", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, function: void (float**, float*, i32)* @do_not_interleave, variables: !2)
|
||||
!5 = !DISubroutineType(types: !2)
|
||||
!6 = distinct !DISubprogram(name: "interleave_not_profitable", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, function: void (float**, float*, i32)* @interleave_not_profitable, variables: !2)
|
||||
!7 = !{i32 2, !"Dwarf Version", i32 4}
|
||||
!8 = !{i32 2, !"Debug Info Version", i32 3}
|
||||
!9 = !{!"clang version 3.8.0 (trunk 250016)"}
|
||||
!10 = !DILocation(line: 4, column: 23, scope: !4)
|
||||
!11 = !DILocation(line: 4, column: 3, scope: !4)
|
||||
!12 = !DILocation(line: 5, column: 17, scope: !4)
|
||||
!13 = !DILocation(line: 5, column: 16, scope: !4)
|
||||
!14 = !DILocation(line: 5, column: 7, scope: !4)
|
||||
!15 = !DILocation(line: 5, column: 14, scope: !4)
|
||||
!16 = distinct !{!16, !17, !18}
|
||||
!17 = !{!"llvm.loop.interleave.count", i32 1}
|
||||
!18 = !{!"llvm.loop.unroll.disable"}
|
||||
!19 = !DILocation(line: 6, column: 1, scope: !4)
|
||||
!20 = !DILocation(line: 11, column: 23, scope: !6)
|
||||
!21 = !DILocation(line: 11, column: 3, scope: !6)
|
||||
!22 = !DILocation(line: 12, column: 17, scope: !6)
|
||||
!23 = !DILocation(line: 12, column: 16, scope: !6)
|
||||
!24 = !DILocation(line: 12, column: 7, scope: !6)
|
||||
!25 = !DILocation(line: 12, column: 14, scope: !6)
|
||||
!26 = distinct !{!26, !18}
|
||||
!27 = !DILocation(line: 13, column: 1, scope: !6)
|
||||
|
||||
!0 = !{i32 2, !"Debug Info Version", i32 3}
|
||||
!1 = !{i32 1, !"PIC Level", i32 2}
|
||||
!2 = !{!"clang version 3.7.0"}
|
||||
!3 = !DILocation(line: 3, column: 20, scope: !4)
|
||||
!4 = distinct !DISubprogram(name: "do_not_interleave", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, function: void (float*, float*, i32)* @do_not_interleave, variables: !7)
|
||||
!5 = !DIFile(filename: "vectorization-remarks-profitable.c", directory: "")
|
||||
!6 = !DISubroutineType(types: !7)
|
||||
!7 = !{}
|
||||
!8 = !DILocation(line: 3, column: 3, scope: !4)
|
||||
!9 = !DILocation(line: 4, column: 14, scope: !4)
|
||||
!10 = !DILocation(line: 6, column: 1, scope: !4)
|
||||
!11 = !{!12, !12, i64 0}
|
||||
!12 = !{!"float", !13, i64 0}
|
||||
!13 = !{!"omnipotent char", !14, i64 0}
|
||||
!14 = !{!"Simple C/C++ TBAA"}
|
||||
!15 = !DILocation(line: 4, column: 5, scope: !4)
|
||||
!16 = !DILocation(line: 4, column: 12, scope: !4)
|
||||
!17 = distinct !{!17, !18, !19}
|
||||
!18 = !{!"llvm.loop.interleave.count", i32 1}
|
||||
!19 = !{!"llvm.loop.unroll.disable"}
|
||||
!20 = !DILocation(line: 10, column: 20, scope: !21)
|
||||
!21 = distinct !DISubprogram(name: "interleave_not_profitable", scope: !5, file: !5, line: 8, type: !6, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, function: void (float*, float*, i32)* @interleave_not_profitable, variables: !7)
|
||||
!22 = !DILocation(line: 10, column: 3, scope: !21)
|
||||
!23 = !DILocation(line: 11, column: 14, scope: !21)
|
||||
!24 = !DILocation(line: 13, column: 1, scope: !21)
|
||||
!25 = !DILocation(line: 11, column: 5, scope: !21)
|
||||
!26 = !DILocation(line: 11, column: 12, scope: !21)
|
||||
!27 = distinct !{!27, !19}
|
||||
|
||||
40
test/Transforms/LoopVectorize/gep_with_bitcast.ll
Normal file
40
test/Transforms/LoopVectorize/gep_with_bitcast.ll
Normal file
@ -0,0 +1,40 @@
|
||||
; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 < %s | FileCheck %s
|
||||
|
||||
target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
|
||||
|
||||
; Vectorization of loop with bitcast between GEP and load
|
||||
; Simplified source code:
|
||||
;void foo (double** __restrict__ in, bool * __restrict__ res) {
|
||||
;
|
||||
; for (int i = 0; i < 4096; ++i)
|
||||
; res[i] = ((unsigned long long)in[i] == 0);
|
||||
;}
|
||||
|
||||
; CHECK-LABEL: @foo
|
||||
; CHECK: vector.body
|
||||
; CHECK: %0 = getelementptr inbounds double*, double** %in, i64 %index
|
||||
; CHECK: %1 = bitcast double** %0 to <4 x i64>*
|
||||
; CHECK: %wide.load = load <4 x i64>, <4 x i64>* %1, align 8
|
||||
; CHECK: %2 = icmp eq <4 x i64> %wide.load, zeroinitializer
|
||||
; CHECK: br i1
|
||||
|
||||
define void @foo(double** noalias nocapture readonly %in, double** noalias nocapture readnone %out, i8* noalias nocapture %res) #0 {
|
||||
entry:
|
||||
br label %for.body
|
||||
|
||||
for.body:
|
||||
%indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
|
||||
%arrayidx = getelementptr inbounds double*, double** %in, i64 %indvars.iv
|
||||
%tmp53 = bitcast double** %arrayidx to i64*
|
||||
%tmp54 = load i64, i64* %tmp53, align 8
|
||||
%cmp1 = icmp eq i64 %tmp54, 0
|
||||
%arrayidx3 = getelementptr inbounds i8, i8* %res, i64 %indvars.iv
|
||||
%frombool = zext i1 %cmp1 to i8
|
||||
store i8 %frombool, i8* %arrayidx3, align 1
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
|
||||
%exitcond = icmp eq i64 %indvars.iv.next, 4096
|
||||
br i1 %exitcond, label %for.end, label %for.body
|
||||
|
||||
for.end:
|
||||
ret void
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user