Magic-Mirror/llvm-capstone
1
0
Fork 0
mirror of https://github.com/capstone-engine/llvm-capstone.git synced 2025-02-18 16:58:23 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[VPlan] Wrap vector loop blocks in region.

Browse Source 
A first step towards modeling preheader and exit blocks in VPlan as well.
Keeping the vector loop in a region allows for changing the VF as we
traverse region boundaries.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D113182
This commit is contained in:
Florian Hahn 2021-11-20 17:59:47 +00:00
parent 337948ac6e
commit cf8efbd30e
No known key found for this signature in database
GPG Key ID: EEF712BB5E80EBBA
8 changed files with 58 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -9354,7 +9354,9 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
if (VPBB)
VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
else {
Plan->setEntry(FirstVPBBForBB);
auto *TopRegion = new VPRegionBlock("vector loop");
TopRegion->setEntry(FirstVPBBForBB);
Plan->setEntry(TopRegion);
HeaderVPBB = FirstVPBBForBB;
}
VPBB = FirstVPBBForBB;
@ -9426,9 +9428,11 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
}
}
assert(isa<VPBasicBlock>(Plan->getEntry()) &&
assert(isa<VPRegionBlock>(Plan->getEntry()) &&
!Plan->getEntry()->getEntryBasicBlock()->empty() &&
"entry block must be set to a non-empty VPBasicBlock");
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
cast<VPRegionBlock>(Plan->getEntry())->setExit(VPBB);
RecipeBuilder.fixHeaderPhis();
// ---------------------------------------------------------------------------

1
llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
View File

@ -8,6 +8,7 @@ target triple = "aarch64-unknown-linux-gnu"
; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, i8* %ptr.iv.2, i64 1
;
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN-PHI %ptr.iv.1 = phi %start.1, %ptr.iv.1.next

15
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
View File

@ -11,6 +11,7 @@ define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
@ -62,6 +63,8 @@ define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize {
; CHECK-NEXT: WIDEN ir<%add> = add ir<%conv>, vp<[[PRED2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
@ -87,6 +90,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
@ -123,6 +127,8 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize {
; CHECK-NEXT: loop.1:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
@ -148,6 +154,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
@ -182,6 +189,8 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]> ir<%and.red.next> ir<%and.red>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
@ -210,6 +219,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8*
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
@ -267,6 +277,8 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8*
; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop
@ -296,6 +308,7 @@ define void @sink_replicate_region_after_replicate_region(i32* %ptr, i32 %x, i8
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
@ -350,6 +363,8 @@ define void @sink_replicate_region_after_replicate_region(i32* %ptr, i32 %x, i8
; CHECK-NEXT: loop.2:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %loop

1
llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
View File

@ -37,6 +37,7 @@ for.end:
; Check for crash exposed by D76992.
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%cond0> = icmp ir<%iv>, ir<13>

5
llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
View File

@ -7,7 +7,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; Verify that -vplan-print-in-dot-format option works.
define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
; CHECK: N0 [label =
; CHECK: subgraph cluster_N0 {
; CHECK-NEXT: fontname=Courier
; CHECK-NEXT: label="\<x1\> vector loop"
; CHECK-NEXT: N1 [label =
; CHECK-NEXT: "for.body:\l" +
; CHECK-NEXT: " WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr ir\<%y\>, ir\<%iv\>\l" +

15
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
View File

@ -9,6 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
; CHECK-LABEL: Checking a loop in "print_call_and_memory"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
@ -18,6 +19,8 @@ define void @print_call_and_memory(i64 %n, float* noalias %y, float* noalias %x)
; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
%cmp6 = icmp sgt i64 %n, 0
@ -41,6 +44,7 @@ for.end: ; preds = %for.body, %entry
define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalias %x, float* %z) nounwind uwtable {
; CHECK-LABEL: Checking a loop in "print_widen_gep_and_select"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
@ -52,6 +56,8 @@ define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalia
; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
%cmp6 = icmp sgt i64 %n, 0
@ -77,6 +83,7 @@ for.end: ; preds = %for.body, %entry
define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK-LABEL: Checking a loop in "print_reduction"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
@ -85,6 +92,8 @@ define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %for.body
@ -106,6 +115,7 @@ for.end: ; preds = %for.body, %entry
define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
; CHECK-LABEL: Checking a loop in "print_replicate_predicated_phi"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next
; CHECK-NEXT: WIDEN ir<%cmp> = icmp ir<%i>, ir<5>
@ -140,6 +150,8 @@ define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %for.body
@ -171,6 +183,7 @@ for.end: ; preds = %for.inc
define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-LABEL: Checking a loop in "print_interleave_groups"
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr ir<@AB>, ir<0>, ir<%iv>
@ -195,6 +208,8 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
; CHECK-NEXT: store ir<%AB.3> to index 3
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %for.body

3
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
View File

@ -8,6 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
define void @sink_with_sideeffects(i1 %c, i8* %ptr) {
; CHECK-LABEL: sink_with_sideeffects
; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %tmp0 = phi %tmp6, 0
; CHECK-NEXT: WIDEN-INDUCTION %tmp1 = phi %tmp7, 0
@ -41,6 +42,8 @@ define void @sink_with_sideeffects(i1 %c, i8* %ptr) {
; CHECK: for.inc:
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: No successors
; CHECK-NEXT: }
;
entry:
br label %for.body

12
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
View File

@ -13,6 +13,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
@ -73,6 +74,7 @@ exit:
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
@ -146,6 +148,7 @@ exit:
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
@ -221,6 +224,7 @@ define void @uniform_gep(i64 %k, i16* noalias %A, i16* noalias %B) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 21, %iv.next
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION
@ -305,6 +309,7 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%c.1> = icmp ir<%iv>, ir<%j>
@ -400,6 +405,7 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
@ -510,6 +516,7 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
@ -614,6 +621,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
@ -723,6 +731,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv> vp<[[BTC]]>
@ -784,6 +793,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, ir<%lv.a>
@ -860,6 +870,7 @@ exit:
define void @update_multiple_users(i16* noalias %src, i8* noalias %dst, i1 %c) {
; CHECK-LABEL: LV: Checking a loop in "update_multiple_users"
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.header:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: Successor(s): loop.then
@ -926,6 +937,7 @@ exit:
define void @sinking_requires_duplication(float* %addr) {
; CHECK-LABEL: LV: Checking a loop in "sinking_requires_duplication"
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: loop.header:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, ir<%iv>