RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-05 18:49:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[AMDGPU] Fix typo in SIInstrInfo::memOpsHaveSameBasePtr

Browse Source 
Summary:
The typo has been present since memOpsHaveSameBasePtr was introduced in
r313208.

It caused SIInstrInfo::shouldClusterMemOps to cluster more mem ops than
it was supposed to.

Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D71616
This commit is contained in:
Jay Foad 2019-12-17 16:09:02 +00:00
parent 49c1dbe690
commit d10f996d89
23 changed files with 3691 additions and 3672 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Target/AMDGPU/SIInstrInfo.cpp
View File

@ -423,7 +423,7 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
const MachineFunction &MF = *MI1.getParent()->getParent();
const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
Base1 = GetUnderlyingObject(Base1, DL);
Base2 = GetUnderlyingObject(Base1, DL);
Base2 = GetUnderlyingObject(Base2, DL);
if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
return false;

4
test/CodeGen/AMDGPU/add.i16.ll
View File

@ -105,7 +105,7 @@ define amdgpu_kernel void @v_test_add_i16_zext_to_i64(i64 addrspace(1)* %out, i1
; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i32:
; VI: flat_load_ushort [[A:v[0-9]+]]
; VI: flat_load_ushort [[B:v[0-9]+]]
; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
; VI-NEXT: buffer_store_dword [[SEXT]]
define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@ -125,7 +125,7 @@ define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i1
; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i64:
; VI: flat_load_ushort [[A:v[0-9]+]]
; VI: flat_load_ushort [[B:v[0-9]+]]
; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}

2
test/CodeGen/AMDGPU/ctpop.ll
View File

@ -284,7 +284,7 @@ define amdgpu_kernel void @v_ctpop_i32_add_var_inv(i32 addrspace(1)* noalias %ou
; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
; VI: flat_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]
; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
; GCN: buffer_store_dword [[RESULT]],
; GCN: s_endpgm

2
test/CodeGen/AMDGPU/ctpop16.ll
View File

@ -285,7 +285,7 @@ define amdgpu_kernel void @v_ctpop_i16_add_var_inv(i16 addrspace(1)* noalias %ou
; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
; VI: flat_load_ushort [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
; VI: flat_load_ushort [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]
; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
; GCN: buffer_store_short [[RESULT]],
; GCN: s_endpgm

2
test/CodeGen/AMDGPU/fadd.f16.ll
View File

@ -62,8 +62,8 @@ entry:
; GCN-LABEL: {{^}}fadd_v2f16:
; SI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
; SI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
; VI: flat_load_dword v[[B_V2_F16:[0-9]+]]
; VI: flat_load_dword v[[A_V2_F16:[0-9]+]]
; VI: flat_load_dword v[[B_V2_F16:[0-9]+]]
; SI-DAG: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
; SI-DAG: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]

2
test/CodeGen/AMDGPU/global_smrd.ll
View File

@ -83,8 +83,8 @@ define amdgpu_kernel void @memdep(i32 addrspace(1)* %in, [8 x i32], i32 addrspac
; CHECK: s_getpc_b64 [[GET_PC:s\[[0-9]+:[0-9]+\]]]
; CHECK: s_load_dwordx2 [[A_ADDR:s\[[0-9]+:[0-9]+\]]], [[GET_PC]], 0x0
; CHECK: s_load_dwordx2 [[A_ADDR1:s\[[0-9]+:[0-9]+\]]], [[A_ADDR]], 0x0
; CHECK: s_load_dword [[SVAL:s[0-9]+]], [[A_ADDR1]], 0x0
; CHECK: s_load_dwordx2 [[OUT:s\[[0-9]+:[0-9]+\]]], s[4:5], 0x0
; CHECK: s_load_dword [[SVAL:s[0-9]+]], [[A_ADDR1]], 0x0
; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]]
@A = common local_unnamed_addr addrspace(1) global i32 addrspace(1)* null, align 4

977
test/CodeGen/AMDGPU/idot2.ll
View File

File diff suppressed because it is too large Load Diff

678
test/CodeGen/AMDGPU/idot4s.ll
View File

@ -16,22 +16,22 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i8 s7, s4
; GFX7-NEXT: s_sext_i32_i8 s8, s5
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v0, s8
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v0, v1
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v1, s10
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s9, v1, v0
; GFX7-NEXT: s_sext_i32_i8 s6, s4
; GFX7-NEXT: s_sext_i32_i8 s7, s5
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v1, s9
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: s_ashr_i32 s5, s5, 24
; GFX7-NEXT: v_mad_i32_i24 v0, s11, v1, v0
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 24
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
@ -45,27 +45,27 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s4, s2
; GFX8-NEXT: s_sext_i32_i8 s5, s3
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v0, s6, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s0, s2
; GFX8-NEXT: s_sext_i32_i8 s1, s3
; GFX8-NEXT: s_bfe_i32 s6, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v2, s1
; GFX8-NEXT: v_mov_b32_e32 v3, s4
; GFX8-NEXT: s_bfe_i32 s8, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v2, v3
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v3, s6
; GFX8-NEXT: s_bfe_i32 s7, s2, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v2, s5, v3, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s8
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mad_i32_i24 v2, s7, v3, v2
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@ -76,27 +76,27 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NODL-NEXT: s_load_dword s10, s[0:1], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s2
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s3
; GFX9-NODL-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s10
; GFX9-NODL-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX9-NODL-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NODL-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s6, v1, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NODL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX9-NODL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s0, s2
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s3
; GFX9-NODL-NEXT: s_bfe_i32 s6, s3, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NODL-NEXT: s_bfe_i32 s8, s3, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s0, v2, v3
; GFX9-NODL-NEXT: s_bfe_i32 s5, s2, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NODL-NEXT: s_bfe_i32 s7, s2, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s5, v3, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s8
; GFX9-NODL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s7, v3, v2
; GFX9-NODL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX9-NODL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NODL-NEXT: s_endpgm
;
@ -105,15 +105,15 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX9-DL-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
; GFX9-DL-NEXT: v_dot4_i32_i8 v2, s2, v2, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-DL-NEXT: v_dot4_i32_i8 v2, s4, v2, v3
; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-DL-NEXT: s_endpgm
;
@ -123,14 +123,14 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX10-DL-NEXT: ; implicit-def: $vcc_hi
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX10-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s2
; GFX10-DL-NEXT: v_dot4_i32_i8 v2, s3, s4, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX10-DL-NEXT: v_dot4_i32_i8 v2, s2, s3, v2
; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
@ -220,29 +220,29 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ushort v2, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s0, s2
; GFX8-NEXT: s_sext_i32_i8 s1, s3
; GFX8-NEXT: s_sext_i32_i8 s1, s2
; GFX8-NEXT: s_bfe_i32 s3, s2, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: s_bfe_i32 s4, s3, 0x80008
; GFX8-NEXT: s_bfe_i32 s5, s3, 0x80010
; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: s_bfe_i32 s1, s2, 0x80008
; GFX8-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s1, s0
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: s_bfe_i32 s4, s0, 0x80008
; GFX8-NEXT: s_bfe_i32 s3, s0, 0x80010
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: s_ashr_i32 s0, s0, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_i32_i24 v2, s1, v3, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v4, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s3, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s2
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s1, v4, v2
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@ -251,29 +251,29 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NODL-NEXT: global_load_ushort v2, v[0:1], off
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s0, s2
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s3
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s2
; GFX9-NODL-NEXT: s_bfe_i32 s3, s2, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NODL-NEXT: s_bfe_i32 s4, s3, 0x80008
; GFX9-NODL-NEXT: s_bfe_i32 s5, s3, 0x80010
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NODL-NEXT: s_bfe_i32 s1, s2, 0x80008
; GFX9-NODL-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX9-NODL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NODL-NEXT: s_bfe_i32 s5, s2, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s3
; GFX9-NODL-NEXT: s_bfe_i32 s4, s0, 0x80008
; GFX9-NODL-NEXT: s_bfe_i32 s3, s0, 0x80010
; GFX9-NODL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NODL-NEXT: s_ashr_i32 s0, s0, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s1, v3, v2
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v4, v2
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s3, v5, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s0, v3, v2
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s1, v4, v2
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v5, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX9-NODL-NEXT: global_store_short v[0:1], v2, off
; GFX9-NODL-NEXT: s_endpgm
;
@ -282,15 +282,15 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_i32_i8 v2, s2, v3, v2
; GFX9-DL-NEXT: v_dot4_i32_i8 v2, s0, v3, v2
; GFX9-DL-NEXT: global_store_short v[0:1], v2, off
; GFX9-DL-NEXT: s_endpgm
;
@ -356,28 +356,28 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_movk_i32 s8, 0xff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX7-NEXT: s_movk_i32 s5, 0xff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_and_b32 s7, s4, s8
; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008
; GFX7-NEXT: s_and_b32 s6, s5, s8
; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010
; GFX7-NEXT: s_lshr_b32 s5, s5, 24
; GFX7-NEXT: s_and_b32 s7, s6, s5
; GFX7-NEXT: s_and_b32 s5, s4, s5
; GFX7-NEXT: s_bfe_u32 s8, s6, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: s_bfe_u32 s10, s6, 0x80010
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: s_lshr_b32 s6, s6, 24
; GFX7-NEXT: v_mov_b32_e32 v3, s10
; GFX7-NEXT: s_lshr_b32 s4, s4, 24
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0
; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0
; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
@ -386,31 +386,31 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_movk_i32 s2, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ubyte v2, v[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX8-NEXT: s_movk_i32 s0, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_bfe_u32 s5, s0, 0x80008
; GFX8-NEXT: s_and_b32 s3, s1, s2
; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010
; GFX8-NEXT: s_and_b32 s3, s1, s0
; GFX8-NEXT: s_and_b32 s0, s2, s0
; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
; GFX8-NEXT: s_and_b32 s2, s0, s2
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010
; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: s_bfe_u32 s7, s0, 0x80010
; GFX8-NEXT: s_lshr_b32 s1, s1, 24
; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: s_lshr_b32 s0, s0, 24
; GFX8-NEXT: s_lshr_b32 s2, s2, 24
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_byte v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@ -418,31 +418,31 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-NODL-NEXT: s_movk_i32 s2, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NODL-NEXT: global_load_ubyte v2, v[0:1], off
; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_bfe_u32 s5, s0, 0x80008
; GFX9-NODL-NEXT: s_and_b32 s3, s1, s2
; GFX9-NODL-NEXT: s_bfe_u32 s5, s2, 0x80008
; GFX9-NODL-NEXT: s_bfe_u32 s7, s2, 0x80010
; GFX9-NODL-NEXT: s_and_b32 s3, s1, s0
; GFX9-NODL-NEXT: s_and_b32 s0, s2, s0
; GFX9-NODL-NEXT: s_bfe_u32 s4, s1, 0x80008
; GFX9-NODL-NEXT: s_and_b32 s2, s0, s2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80010
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NODL-NEXT: s_bfe_u32 s7, s0, 0x80010
; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24
; GFX9-NODL-NEXT: s_lshr_b32 s2, s2, 24
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v4, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s7, v5, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s2, v3, v2
; GFX9-NODL-NEXT: global_store_byte v[0:1], v2, off
; GFX9-NODL-NEXT: s_endpgm
;
@ -451,15 +451,15 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off
; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s2, v3, v2
; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s0, v3, v2
; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off
; GFX9-DL-NEXT: s_endpgm
;
@ -520,23 +520,23 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_sext_i32_i8 s7, s4
; GFX7-NEXT: s_sext_i32_i8 s8, s5
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v0, s8
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x80008
; GFX7-NEXT: v_mad_i32_i24 v1, s7, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s10
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v1, s9, v2, v1
; GFX7-NEXT: s_bfe_i32 s11, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v0, v1
; GFX7-NEXT: s_sext_i32_i8 s6, s4
; GFX7-NEXT: s_sext_i32_i8 s7, s5
; GFX7-NEXT: s_bfe_i32 s9, s5, 0x80008
; GFX7-NEXT: v_mov_b32_e32 v0, s7
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
; GFX7-NEXT: v_mad_i32_i24 v1, s6, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s9
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v1, s8, v2, v1
; GFX7-NEXT: s_bfe_i32 s10, s4, 0x80010
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: s_ashr_i32 s5, s5, 24
; GFX7-NEXT: v_mad_i32_i24 v0, s11, v1, v0
; GFX7-NEXT: v_mad_i32_i24 v0, s10, v1, v0
; GFX7-NEXT: s_ashr_i32 s4, s4, 24
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v1, v0
@ -550,28 +550,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s4, s2
; GFX8-NEXT: s_sext_i32_i8 s5, s3
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s10
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX8-NEXT: v_mad_i32_i24 v1, s4, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s7
; GFX8-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX8-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_sext_i32_i8 s0, s2
; GFX8-NEXT: s_sext_i32_i8 s1, s3
; GFX8-NEXT: s_bfe_i32 s6, s3, 0x80008
; GFX8-NEXT: v_mov_b32_e32 v2, s1
; GFX8-NEXT: v_mov_b32_e32 v3, s4
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80008
; GFX8-NEXT: v_mad_i32_i24 v3, s0, v2, v3
; GFX8-NEXT: v_mov_b32_e32 v4, s6
; GFX8-NEXT: s_bfe_i32 s8, s3, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v3, s5, v4, v3
; GFX8-NEXT: s_bfe_i32 s7, s2, 0x80010
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v2, v3
; GFX8-NEXT: v_mov_b32_e32 v3, s8
; GFX8-NEXT: s_ashr_i32 s3, s3, 24
; GFX8-NEXT: v_mad_i32_i24 v2, s7, v3, v2
; GFX8-NEXT: s_ashr_i32 s2, s2, 24
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@ -582,28 +582,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NODL-NEXT: s_load_dword s10, s[0:1], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s2
; GFX9-NODL-NEXT: s_sext_i32_i8 s5, s3
; GFX9-NODL-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s10
; GFX9-NODL-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s4, v0, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-NODL-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX9-NODL-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NODL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX9-NODL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_sext_i32_i8 s0, s2
; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s3
; GFX9-NODL-NEXT: s_bfe_i32 s6, s3, 0x80008
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NODL-NEXT: s_bfe_i32 s5, s2, 0x80008
; GFX9-NODL-NEXT: v_mad_i32_i24 v3, s0, v2, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NODL-NEXT: s_bfe_i32 s8, s3, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v3, s5, v4, v3
; GFX9-NODL-NEXT: s_bfe_i32 s7, s2, 0x80010
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s0, v2, v3
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s8
; GFX9-NODL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s7, v3, v2
; GFX9-NODL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX9-NODL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NODL-NEXT: s_endpgm
;
@ -614,28 +614,28 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DL-NEXT: s_load_dword s10, s[0:1], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i8 s4, s2
; GFX9-DL-NEXT: s_sext_i32_i8 s5, s3
; GFX9-DL-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s5
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s10
; GFX9-DL-NEXT: s_bfe_i32 s6, s2, 0x80008
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s4, v0, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-DL-NEXT: s_bfe_i32 s9, s3, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v1, s6, v2, v1
; GFX9-DL-NEXT: s_bfe_i32 s8, s2, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s9
; GFX9-DL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-DL-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX9-DL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s2, v1, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_sext_i32_i8 s0, s2
; GFX9-DL-NEXT: s_sext_i32_i8 s1, s3
; GFX9-DL-NEXT: s_bfe_i32 s6, s3, 0x80008
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s1
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4
; GFX9-DL-NEXT: s_bfe_i32 s5, s2, 0x80008
; GFX9-DL-NEXT: v_mad_i32_i24 v3, s0, v2, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s6
; GFX9-DL-NEXT: s_bfe_i32 s8, s3, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v3, s5, v4, v3
; GFX9-DL-NEXT: s_bfe_i32 s7, s2, 0x80010
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s0, v2, v3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s8
; GFX9-DL-NEXT: s_ashr_i32 s3, s3, 24
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s7, v3, v2
; GFX9-DL-NEXT: s_ashr_i32 s2, s2, 24
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s2, v3, v2
; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-DL-NEXT: s_endpgm
;
@ -648,23 +648,23 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX10-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_sext_i32_i8 s5, s2
; GFX10-DL-NEXT: s_sext_i32_i8 s6, s3
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80008
; GFX10-DL-NEXT: s_bfe_i32 s7, s3, 0x80008
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s6, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s7, v0
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX10-DL-NEXT: s_bfe_i32 s7, s3, 0x80010
; GFX10-DL-NEXT: s_ashr_i32 s2, s2, 24
; GFX10-DL-NEXT: s_ashr_i32 s3, s3, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s5, s6, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s7, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s2, s3, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_sext_i32_i8 s0, s2
; GFX10-DL-NEXT: s_sext_i32_i8 s1, s3
; GFX10-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80008
; GFX10-DL-NEXT: s_bfe_i32 s5, s3, 0x80008
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s0, s1, v2
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX10-DL-NEXT: s_bfe_i32 s5, s3, 0x80010
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s0, s1, v2
; GFX10-DL-NEXT: s_ashr_i32 s0, s2, 24
; GFX10-DL-NEXT: s_ashr_i32 s1, s3, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s0, s1, v2
; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
@ -719,25 +719,25 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_ashr_i32 s7, s4, 24
; GFX7-NEXT: s_ashr_i32 s10, s5, 24
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80010
; GFX7-NEXT: s_bfe_i32 s12, s5, 0x80008
; GFX7-NEXT: s_ashr_i32 s6, s4, 24
; GFX7-NEXT: s_ashr_i32 s9, s5, 24
; GFX7-NEXT: s_bfe_i32 s10, s5, 0x80010
; GFX7-NEXT: s_bfe_i32 s11, s5, 0x80008
; GFX7-NEXT: s_sext_i32_i8 s5, s5
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80010
; GFX7-NEXT: s_bfe_i32 s9, s4, 0x80008
; GFX7-NEXT: s_bfe_i32 s7, s4, 0x80010
; GFX7-NEXT: s_bfe_i32 s8, s4, 0x80008
; GFX7-NEXT: s_sext_i32_i8 s4, s4
; GFX7-NEXT: v_mov_b32_e32 v0, s5
; GFX7-NEXT: v_mov_b32_e32 v1, s6
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: v_mad_i32_i24 v0, s9, v1, v0
; GFX7-NEXT: v_mad_i32_i24 v0, s4, v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s11
; GFX7-NEXT: v_mad_i32_i24 v0, s8, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s10
; GFX7-NEXT: v_mad_i32_i24 v0, s7, v1, v0
; GFX7-NEXT: v_mov_b32_e32 v1, s9
; GFX7-NEXT: v_mad_i32_i24 v0, s6, v1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
@ -748,28 +748,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX8-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s2
; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s3
; GFX8-NEXT: s_ashr_i32 s6, s3, 24
; GFX8-NEXT: s_bfe_i32 s7, s3, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s3, s3
; GFX8-NEXT: s_ashr_i32 s4, s2, 24
; GFX8-NEXT: s_bfe_i32 s5, s2, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s2, s2
; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: v_mov_b32_e32 v3, s8
; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX8-NEXT: v_mad_i32_i24 v2, s2, v2, v3
; GFX8-NEXT: v_mad_i32_i24 v0, v0, v1, v2
; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mad_i32_i24 v0, s5, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: v_mad_i32_i24 v2, s4, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b16_e64 v2, 8, s2
; GFX8-NEXT: v_lshrrev_b16_e64 v3, 8, s3
; GFX8-NEXT: s_ashr_i32 s5, s3, 24
; GFX8-NEXT: s_bfe_i32 s6, s3, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s3, s3
; GFX8-NEXT: s_ashr_i32 s0, s2, 24
; GFX8-NEXT: s_bfe_i32 s1, s2, 0x80010
; GFX8-NEXT: s_sext_i32_i8 s2, s2
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX8-NEXT: v_mad_i32_i24 v4, s2, v4, v5
; GFX8-NEXT: v_mad_i32_i24 v2, v2, v3, v4
; GFX8-NEXT: v_mov_b32_e32 v3, s6
; GFX8-NEXT: v_mad_i32_i24 v2, s1, v3, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: v_mad_i32_i24 v2, s0, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@ -780,28 +780,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-NODL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NODL-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v0, 8, s2
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s3
; GFX9-NODL-NEXT: s_ashr_i32 s6, s3, 24
; GFX9-NODL-NEXT: s_bfe_i32 s7, s3, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s3, s3
; GFX9-NODL-NEXT: s_ashr_i32 s4, s2, 24
; GFX9-NODL-NEXT: s_bfe_i32 s5, s2, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s2, s2
; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s8
; GFX9-NODL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s2, v2, v3
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, v0, v1, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NODL-NEXT: v_mad_i32_i24 v0, s5, v1, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v1, v0
; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v2, 8, s2
; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v3, 8, s3
; GFX9-NODL-NEXT: s_ashr_i32 s5, s3, 24
; GFX9-NODL-NEXT: s_bfe_i32 s6, s3, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s3, s3
; GFX9-NODL-NEXT: s_ashr_i32 s0, s2, 24
; GFX9-NODL-NEXT: s_bfe_i32 s1, s2, 0x80010
; GFX9-NODL-NEXT: s_sext_i32_i8 s2, s2
; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s3
; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s4
; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-NODL-NEXT: v_mad_i32_i24 v4, s2, v4, v5
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v2, v3, v4
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s1, v3, v2
; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s0, v3, v2
; GFX9-NODL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NODL-NEXT: s_endpgm
;
@ -812,28 +812,28 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DL-NEXT: s_load_dword s8, s[0:1], 0x0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v0, 8, s2
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s3
; GFX9-DL-NEXT: s_ashr_i32 s6, s3, 24
; GFX9-DL-NEXT: s_bfe_i32 s7, s3, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s3, s3
; GFX9-DL-NEXT: s_ashr_i32 s4, s2, 24
; GFX9-DL-NEXT: s_bfe_i32 s5, s2, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s2, s2
; GFX9-DL-NEXT: v_mov_b32_e32 v2, s3
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s8
; GFX9-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s2, v2, v3
; GFX9-DL-NEXT: v_mad_i32_i24 v0, v0, v1, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7
; GFX9-DL-NEXT: v_mad_i32_i24 v0, s5, v1, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s6
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v1, v0
; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s2
; GFX9-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s3
; GFX9-DL-NEXT: s_ashr_i32 s5, s3, 24
; GFX9-DL-NEXT: s_bfe_i32 s6, s3, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s3, s3
; GFX9-DL-NEXT: s_ashr_i32 s0, s2, 24
; GFX9-DL-NEXT: s_bfe_i32 s1, s2, 0x80010
; GFX9-DL-NEXT: s_sext_i32_i8 s2, s2
; GFX9-DL-NEXT: v_mov_b32_e32 v4, s3
; GFX9-DL-NEXT: v_mov_b32_e32 v5, s4
; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX9-DL-NEXT: v_mad_i32_i24 v4, s2, v4, v5
; GFX9-DL-NEXT: v_mad_i32_i24 v2, v2, v3, v4
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s6
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s1, v3, v2
; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-DL-NEXT: v_mad_i32_i24 v2, s0, v3, v2
; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX9-DL-NEXT: s_endpgm
;
@ -846,24 +846,24 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX10-DL-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v0, 8, s2
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s3
; GFX10-DL-NEXT: v_mov_b32_e32 v2, s4
; GFX10-DL-NEXT: s_sext_i32_i8 s5, s2
; GFX10-DL-NEXT: s_sext_i32_i8 s6, s3
; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX10-DL-NEXT: s_ashr_i32 s2, s2, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s5, s6, v2
; GFX10-DL-NEXT: s_bfe_i32 s5, s3, 0x80010
; GFX10-DL-NEXT: s_ashr_i32 s3, s3, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v1, v2
; GFX10-DL-NEXT: v_mad_i32_i24 v0, s4, s5, v0
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s2, s3, v0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v2, 8, s2
; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s3
; GFX10-DL-NEXT: v_mov_b32_e32 v4, s4
; GFX10-DL-NEXT: s_sext_i32_i8 s0, s2
; GFX10-DL-NEXT: s_sext_i32_i8 s1, s3
; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX10-DL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX10-DL-NEXT: s_bfe_i32 s4, s2, 0x80010
; GFX10-DL-NEXT: s_bfe_i32 s5, s3, 0x80010
; GFX10-DL-NEXT: v_mad_i32_i24 v4, s0, s1, v4
; GFX10-DL-NEXT: s_ashr_i32 s0, s2, 24
; GFX10-DL-NEXT: s_ashr_i32 s1, s3, 24
; GFX10-DL-NEXT: v_mad_i32_i24 v2, v2, v3, v4
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2
; GFX10-DL-NEXT: v_mad_i32_i24 v2, s0, s1, v2
; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off
; GFX10-DL-NEXT: s_endpgm
<4 x i8> addrspace(1)* %src2,
@ -927,33 +927,33 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX8-NEXT: s_mov_b32 s2, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_ushort v2, v[0:1]
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0
; GFX8-NEXT: s_mov_b32 s0, 0xffff
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_bfe_i32 s6, s2, 0x80000
; GFX8-NEXT: s_lshr_b32 s4, s2, 16
; GFX8-NEXT: s_bfe_i32 s5, s1, 0x80000
; GFX8-NEXT: v_ashrrev_i16_e64 v4, 8, s1
; GFX8-NEXT: s_bfe_i32 s1, s3, 0x80000
; GFX8-NEXT: v_ashrrev_i16_e64 v6, 8, s3
; GFX8-NEXT: s_and_b32 s3, s0, s6
; GFX8-NEXT: v_ashrrev_i16_e64 v3, 8, s2
; GFX8-NEXT: s_bfe_i32 s2, s4, 0x80000
; GFX8-NEXT: s_bfe_i32 s6, s3, 0x80000
; GFX8-NEXT: s_lshr_b32 s4, s3, 16
; GFX8-NEXT: v_ashrrev_i16_e64 v3, 8, s3
; GFX8-NEXT: s_bfe_i32 s3, s4, 0x80000
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
; GFX8-NEXT: s_bfe_i32 s5, s0, 0x80000
; GFX8-NEXT: v_ashrrev_i16_e64 v4, 8, s0
; GFX8-NEXT: s_bfe_i32 s0, s1, 0x80000
; GFX8-NEXT: v_ashrrev_i16_e64 v6, 8, s1
; GFX8-NEXT: s_and_b32 s1, s2, s6
; GFX8-NEXT: v_ashrrev_i16_e64 v5, 8, s4
; GFX8-NEXT: s_and_b32 s4, s0, s5
; GFX8-NEXT: v_mov_b32_e32 v7, s3
; GFX8-NEXT: s_and_b32 s2, s0, s2
; GFX8-NEXT: s_and_b32 s0, s0, s1
; GFX8-NEXT: s_and_b32 s4, s2, s5
; GFX8-NEXT: v_mov_b32_e32 v7, s1
; GFX8-NEXT: s_and_b32 s3, s2, s3
; GFX8-NEXT: s_and_b32 s0, s2, s0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mad_u32_u24 v2, s4, v7, v2
; GFX8-NEXT: v_mad_u32_u24 v2, v4, v3, v2
; GFX8-NEXT: v_mov_b32_e32 v3, s2
; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2
; GFX8-NEXT: v_mad_u32_u24 v2, v6, v5, v2
; GFX8-NEXT: flat_store_short v[0:1], v2
@ -1046,31 +1046,31 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX10-DL-NEXT: ; implicit-def: $vcc_hi
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0
; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0
; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1
; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0
; GFX10-DL-NEXT: global_load_ushort v3, v[0:1], off
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DL-NEXT: s_bfe_i32 s4, s0, 0x80000
; GFX10-DL-NEXT: s_bfe_i32 s3, s1, 0x80000
; GFX10-DL-NEXT: s_lshr_b32 s2, s0, 16
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s0
; GFX10-DL-NEXT: s_lshr_b32 s5, s1, 16
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, s1
; GFX10-DL-NEXT: v_and_b32_e32 v6, s3, v2
; GFX10-DL-NEXT: v_and_b32_e32 v7, s4, v2
; GFX10-DL-NEXT: s_bfe_i32 s0, s2, 0x80000
; GFX10-DL-NEXT: s_bfe_i32 s1, s3, 0x80000
; GFX10-DL-NEXT: s_lshr_b32 s4, s2, 16
; GFX10-DL-NEXT: s_lshr_b32 s5, s3, 16
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v4, 8, s2
; GFX10-DL-NEXT: v_and_b32_e32 v7, s0, v2
; GFX10-DL-NEXT: v_and_b32_e32 v6, s1, v2
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v5, 8, s3
; GFX10-DL-NEXT: s_bfe_i32 s0, s4, 0x80000
; GFX10-DL-NEXT: s_bfe_i32 s1, s5, 0x80000
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v7
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v8, 8, s4
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v11, 8, s2
; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v6
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v6, 8, s5
; GFX10-DL-NEXT: v_and_b32_e32 v7, s1, v2
; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v7
; GFX10-DL-NEXT: v_ashrrev_i16_e64 v7, 8, s5
; GFX10-DL-NEXT: v_and_b32_e32 v8, s1, v2
; GFX10-DL-NEXT: v_and_b32_e32 v2, s0, v2
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5
; GFX10-DL-NEXT: v_lshl_or_b32 v5, v6, 16, v7
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v8, 16, v2
; GFX10-DL-NEXT: v_lshl_or_b32 v5, v7, 16, v8
; GFX10-DL-NEXT: v_lshl_or_b32 v2, v11, 16, v2
; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v5
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v4, v3

914
test/CodeGen/AMDGPU/idot4u.ll
View File

File diff suppressed because it is too large Load Diff

1877
test/CodeGen/AMDGPU/idot8s.ll
View File

File diff suppressed because it is too large Load Diff

2472
test/CodeGen/AMDGPU/idot8u.ll
View File

File diff suppressed because it is too large Load Diff

31
test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
View File

@ -562,31 +562,32 @@ define amdgpu_kernel void @maxnum_v4f16(
; SI-NEXT: s_mov_b32 s0, s4
; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v0, s4
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
; SI-NEXT: s_lshr_b32 s4, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s4
; SI-NEXT: s_lshr_b32 s4, s7, 16
; SI-NEXT: v_cvt_f32_f16_e32 v5, s4
; SI-NEXT: v_cvt_f32_f16_e32 v1, s5
; SI-NEXT: s_lshr_b32 s4, s6, 16
; SI-NEXT: v_cvt_f32_f16_e32 v7, s7
; SI-NEXT: v_cvt_f32_f16_e32 v6, s4
; SI-NEXT: v_cvt_f32_f16_e32 v4, s6
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: s_lshr_b32 s5, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
; SI-NEXT: v_cvt_f32_f16_e32 v3, s5
; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v5, s6
; SI-NEXT: v_cvt_f32_f16_e32 v4, s4
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: v_cvt_f32_f16_e32 v7, s5
; SI-NEXT: v_cvt_f32_f16_e32 v6, s4
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
; SI-NEXT: v_max_f32_e32 v3, v3, v5
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v7
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_max_f32_e32 v1, v1, v5
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v6
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_max_f32_e32 v2, v2, v5
; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_max_f32_e32 v0, v0, v4

31
test/CodeGen/AMDGPU/llvm.minnum.f16.ll
View File

@ -615,31 +615,32 @@ define amdgpu_kernel void @minnum_v4f16(
; SI-NEXT: s_mov_b32 s0, s4
; SI-NEXT: s_mov_b32 s1, s5
; SI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_cvt_f32_f16_e32 v0, s4
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
; SI-NEXT: s_lshr_b32 s4, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v3, s4
; SI-NEXT: s_lshr_b32 s4, s7, 16
; SI-NEXT: v_cvt_f32_f16_e32 v5, s4
; SI-NEXT: v_cvt_f32_f16_e32 v1, s5
; SI-NEXT: s_lshr_b32 s4, s6, 16
; SI-NEXT: v_cvt_f32_f16_e32 v7, s7
; SI-NEXT: v_cvt_f32_f16_e32 v6, s4
; SI-NEXT: v_cvt_f32_f16_e32 v4, s6
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: s_lshr_b32 s5, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v2, s4
; SI-NEXT: v_cvt_f32_f16_e32 v3, s5
; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_mul_f32_e32 v3, 1.0, v3
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s6, s5, 16
; SI-NEXT: v_cvt_f32_f16_e32 v5, s6
; SI-NEXT: v_cvt_f32_f16_e32 v4, s4
; SI-NEXT: s_lshr_b32 s4, s4, 16
; SI-NEXT: v_cvt_f32_f16_e32 v7, s5
; SI-NEXT: v_cvt_f32_f16_e32 v6, s4
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v5
; SI-NEXT: v_min_f32_e32 v3, v3, v5
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v7
; SI-NEXT: v_mul_f32_e32 v1, 1.0, v1
; SI-NEXT: v_min_f32_e32 v1, v1, v5
; SI-NEXT: v_mul_f32_e32 v5, 1.0, v6
; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2
; SI-NEXT: v_min_f32_e32 v2, v2, v5
; SI-NEXT: v_mul_f32_e32 v4, 1.0, v4
; SI-NEXT: v_mul_f32_e32 v0, 1.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
; SI-NEXT: v_min_f32_e32 v0, v0, v4

4
test/CodeGen/AMDGPU/madak.ll
View File

@ -10,8 +10,8 @@ declare float @llvm.fabs.f32(float) nounwind readnone
; GCN-LABEL: {{^}}madak_f32:
; GFX6: buffer_load_dword [[VA:v[0-9]+]]
; GFX6: buffer_load_dword [[VB:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VA:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX9: {{flat|global}}_load_dword [[VA:v[0-9]+]]
; GFX9: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX10: {{flat|global}}_load_dword [[VA:v[0-9]+]]
@ -101,8 +101,8 @@ define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %o
; GCN-LABEL: {{^}}madak_inline_imm_f32:
; GFX6: buffer_load_dword [[VA:v[0-9]+]]
; GFX6: buffer_load_dword [[VB:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VA:v[0-9]+]]
; GFX8: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX9: {{flat|global}}_load_dword [[VA:v[0-9]+]]
; GFX9: {{flat|global}}_load_dword [[VB:v[0-9]+]]
; GFX10: {{flat|global}}_load_dword [[VA:v[0-9]+]]

123
test/CodeGen/AMDGPU/max.i16.ll
View File

@ -16,13 +16,13 @@ define amdgpu_kernel void @v_test_imax_sge_i16(i16 addrspace(1)* %out, i16 addrs
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v5, v[0:1]
; VI-NEXT: flat_load_ushort v2, v[2:3]
; VI-NEXT: flat_load_ushort v3, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_i16_e32 v2, v3, v2
; VI-NEXT: v_max_i16_e32 v2, v5, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -38,13 +38,13 @@ define amdgpu_kernel void @v_test_imax_sge_i16(i16 addrspace(1)* %out, i16 addrs
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_ushort v5, v[0:1], off
; GFX9-NEXT: global_load_ushort v2, v[2:3], off
; GFX9-NEXT: global_load_ushort v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i16_e32 v2, v3, v2
; GFX9-NEXT: v_max_i16_e32 v2, v5, v2
; GFX9-NEXT: global_store_short v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -73,15 +73,15 @@ define amdgpu_kernel void @v_test_imax_sge_v2i16(<2 x i16> addrspace(1)* %out, <
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_dword v5, v[0:1]
; VI-NEXT: flat_load_dword v2, v[2:3]
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_i16_e32 v4, v3, v2
; VI-NEXT: v_max_i16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v2, v4, v2
; VI-NEXT: v_max_i16_e32 v3, v5, v2
; VI-NEXT: v_max_i16_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -97,13 +97,13 @@ define amdgpu_kernel void @v_test_imax_sge_v2i16(<2 x i16> addrspace(1)* %out, <
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v5, v[0:1], off
; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_max_i16 v2, v3, v2
; GFX9-NEXT: v_pk_max_i16 v2, v5, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -124,35 +124,35 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(<3 x i16> addrspace(1)* %out, <
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT: v_lshlrev_b32_e32 v8, 3, v0
; VI-NEXT: v_lshlrev_b32_e32 v6, 3, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s7
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v8
; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: v_add_u32_e32 v0, vcc, s6, v6
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, s6, v8
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v6
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v6, vcc, 4, v2
; VI-NEXT: flat_load_dword v9, v[0:1]
; VI-NEXT: v_addc_u32_e32 v7, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v4, v[4:5]
; VI-NEXT: flat_load_dword v5, v[2:3]
; VI-NEXT: flat_load_ushort v6, v[6:7]
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v8
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: v_add_u32_e32 v2, vcc, 4, v0
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v7, s5
; VI-NEXT: v_add_u32_e32 v6, vcc, s4, v6
; VI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; VI-NEXT: flat_load_dword v8, v[0:1]
; VI-NEXT: flat_load_ushort v9, v[4:5]
; VI-NEXT: v_add_u32_e32 v0, vcc, 4, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; VI-NEXT: flat_load_dword v2, v[2:3]
; VI-NEXT: flat_load_ushort v0, v[0:1]
; VI-NEXT: v_add_u32_e32 v4, vcc, 4, v6
; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc
; VI-NEXT: s_waitcnt vmcnt(1) lgkmcnt(1)
; VI-NEXT: v_max_i16_e32 v7, v5, v9
; VI-NEXT: v_max_i16_sdwa v5, v5, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_max_i16_e32 v1, v8, v2
; VI-NEXT: v_max_i16_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v1, v1, v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_i16_e32 v4, v6, v4
; VI-NEXT: v_or_b32_e32 v5, v7, v5
; VI-NEXT: flat_store_dword v[0:1], v5
; VI-NEXT: flat_store_short v[2:3], v4
; VI-NEXT: v_max_i16_e32 v0, v9, v0
; VI-NEXT: flat_store_dword v[6:7], v1
; VI-NEXT: flat_store_short v[4:5], v0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_imax_sge_v3i16:
@ -167,19 +167,20 @@ define amdgpu_kernel void @v_test_imax_sge_v3i16(<3 x i16> addrspace(1)* %out, <
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v6, v[2:3], off
; GFX9-NEXT: global_load_dword v7, v[0:1], off
; GFX9-NEXT: global_load_dword v6, v[0:1], off
; GFX9-NEXT: global_load_dword v7, v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v8, v6
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v8, v7
; GFX9-NEXT: v_pk_max_i16 v7, v7, v6
; GFX9-NEXT: global_load_short_d16 v6, v[2:3], off offset:4
; GFX9-NEXT: v_pk_max_i16 v6, v6, v7
; GFX9-NEXT: global_load_short_d16 v7, v[2:3], off offset:4
; GFX9-NEXT: global_load_short_d16 v8, v[0:1], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_max_i16 v0, v8, v6
; GFX9-NEXT: global_store_dword v[4:5], v7, off
; GFX9-NEXT: v_pk_max_i16 v0, v8, v7
; GFX9-NEXT: global_store_dword v[4:5], v6, off
; GFX9-NEXT: global_store_short v[4:5], v0, off offset:4
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -208,8 +209,8 @@ define amdgpu_kernel void @v_test_imax_sge_v4i16(<4 x i16> addrspace(1)* %out, <
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: v_add_u32_e32 v4, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
@ -235,8 +236,8 @@ define amdgpu_kernel void @v_test_imax_sge_v4i16(<4 x i16> addrspace(1)* %out, <
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
@ -271,13 +272,13 @@ define amdgpu_kernel void @v_test_imax_sgt_i16(i16 addrspace(1)* %out, i16 addrs
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v5, v[0:1]
; VI-NEXT: flat_load_ushort v2, v[2:3]
; VI-NEXT: flat_load_ushort v3, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_i16_e32 v2, v3, v2
; VI-NEXT: v_max_i16_e32 v2, v5, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -293,13 +294,13 @@ define amdgpu_kernel void @v_test_imax_sgt_i16(i16 addrspace(1)* %out, i16 addrs
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_ushort v5, v[0:1], off
; GFX9-NEXT: global_load_ushort v2, v[2:3], off
; GFX9-NEXT: global_load_ushort v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_i16_e32 v2, v3, v2
; GFX9-NEXT: v_max_i16_e32 v2, v5, v2
; GFX9-NEXT: global_store_short v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -328,13 +329,13 @@ define amdgpu_kernel void @v_test_umax_uge_i16(i16 addrspace(1)* %out, i16 addrs
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v5, v[0:1]
; VI-NEXT: flat_load_ushort v2, v[2:3]
; VI-NEXT: flat_load_ushort v3, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_u16_e32 v2, v3, v2
; VI-NEXT: v_max_u16_e32 v2, v5, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -350,13 +351,13 @@ define amdgpu_kernel void @v_test_umax_uge_i16(i16 addrspace(1)* %out, i16 addrs
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_ushort v5, v[0:1], off
; GFX9-NEXT: global_load_ushort v2, v[2:3], off
; GFX9-NEXT: global_load_ushort v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u16_e32 v2, v3, v2
; GFX9-NEXT: v_max_u16_e32 v2, v5, v2
; GFX9-NEXT: global_store_short v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -385,13 +386,13 @@ define amdgpu_kernel void @v_test_umax_ugt_i16(i16 addrspace(1)* %out, i16 addrs
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_ushort v5, v[0:1]
; VI-NEXT: flat_load_ushort v2, v[2:3]
; VI-NEXT: flat_load_ushort v3, v[0:1]
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_u16_e32 v2, v3, v2
; VI-NEXT: v_max_u16_e32 v2, v5, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -407,13 +408,13 @@ define amdgpu_kernel void @v_test_umax_ugt_i16(i16 addrspace(1)* %out, i16 addrs
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_ushort v5, v[0:1], off
; GFX9-NEXT: global_load_ushort v2, v[2:3], off
; GFX9-NEXT: global_load_ushort v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_u16_e32 v2, v3, v2
; GFX9-NEXT: v_max_u16_e32 v2, v5, v2
; GFX9-NEXT: global_store_short v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@ -441,15 +442,15 @@ define amdgpu_kernel void @v_test_umax_ugt_v2i16(<2 x i16> addrspace(1)* %out, <
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v4
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_dword v5, v[0:1]
; VI-NEXT: flat_load_dword v2, v[2:3]
; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v4
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_max_u16_e32 v4, v3, v2
; VI-NEXT: v_max_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v2, v4, v2
; VI-NEXT: v_max_u16_e32 v3, v5, v2
; VI-NEXT: v_max_u16_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@ -465,13 +466,13 @@ define amdgpu_kernel void @v_test_umax_ugt_v2i16(<2 x i16> addrspace(1)* %out, <
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v4
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v5, v[0:1], off
; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v3, v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_max_u16 v2, v3, v2
; GFX9-NEXT: v_pk_max_u16 v2, v5, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone

6
test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
View File

@ -393,11 +393,11 @@ define amdgpu_kernel void @DiffBase(i8 addrspace(1)* %buffer1,
; GFX8: flat_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]
;
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-4096
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
; GFX9: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:2048
;
; GFX10: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:-2048
; GFX10: global_load_dwordx2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}

2
test/CodeGen/AMDGPU/sdwa-peephole.ll
View File

@ -73,7 +73,7 @@ entry:
; GCN-LABEL: {{^}}mul_v2i16:
; NOSDWA: v_lshrrev_b32_e32 v[[DST0:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_lshrrev_b32_e32 v[[DST1:[0-9]+]], 16, v{{[0-9]+}}
; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST0]], v[[DST1]]
; NOSDWA: v_mul_u32_u24_e32 v[[DST_MUL:[0-9]+]], v[[DST1]], v[[DST0]]
; NOSDWA: v_lshlrev_b32_e32 v[[DST_SHL:[0-9]+]], 16, v[[DST_MUL]]
; NOSDWA: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[DST_SHL]]
; NOSDWA-NOT: v_mul_u32_u24_sdwa

2
test/CodeGen/AMDGPU/shl_add_ptr.ll
View File

@ -78,7 +78,7 @@ define amdgpu_kernel void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i
; GCN: v_lshlrev_b32_e32 [[OFS:v[0-9]+]], 2, {{v[0-9]+}}
; GCN-DAG: v_add_{{[iu]}}32_e32 [[PTR0:v[0-9]+]], vcc, lds0@abs32@lo, [[OFS]]
; GCN-DAG: v_add_{{[iu]}}32_e32 [[PTR1:v[0-9]+]], vcc, lds1@abs32@lo, [[OFS]]
; GCN: s_mov_b32 m0, -1
; GCN-DAG: s_mov_b32 m0, -1
; GCN-DAG: ds_read_b32 {{v[0-9]+}}, [[PTR0]] offset:256
; GCN-DAG: ds_read_b32 {{v[0-9]+}}, [[PTR1]] offset:256

4
test/CodeGen/AMDGPU/sub.i16.ll
View File

@ -107,7 +107,7 @@ define amdgpu_kernel void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i1
; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i32:
; VI: flat_load_ushort [[A:v[0-9]+]]
; VI: flat_load_ushort [[B:v[0-9]+]]
; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
; VI-NEXT: buffer_store_dword [[SEXT]]
define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@ -127,7 +127,7 @@ define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i1
; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i64:
; VI: flat_load_ushort [[A:v[0-9]+]]
; VI: flat_load_ushort [[B:v[0-9]+]]
; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}

40
test/CodeGen/AMDGPU/sub.v2i16.ll
View File

@ -66,39 +66,39 @@ define amdgpu_kernel void @s_test_sub_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
; GFX9-LABEL: s_test_sub_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x34
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s6, s[6:7], 0x0
; GFX9-NEXT: s_load_dword s7, s[0:1], 0x0
; GFX9-NEXT: s_mov_b32 s0, s4
; GFX9-NEXT: s_mov_b32 s1, s5
; GFX9-NEXT: s_load_dword s4, s[6:7], 0x0
; GFX9-NEXT: s_load_dword s5, s[8:9], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s7
; GFX9-NEXT: v_pk_sub_i16 v0, s6, v0
; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: v_pk_sub_i16 v0, s4, v0
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; VI-LABEL: s_test_sub_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x34
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_load_dword s6, s[6:7], 0x0
; VI-NEXT: s_load_dword s7, s[0:1], 0x0
; VI-NEXT: s_mov_b32 s0, s4
; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_load_dword s4, s[6:7], 0x0
; VI-NEXT: s_load_dword s5, s[8:9], 0x0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshr_b32 s4, s6, 16
; VI-NEXT: s_lshr_b32 s5, s7, 16
; VI-NEXT: s_lshr_b32 s6, s4, 16
; VI-NEXT: s_lshr_b32 s7, s5, 16
; VI-NEXT: s_sub_i32 s4, s4, s5
; VI-NEXT: s_sub_i32 s6, s6, s7
; VI-NEXT: s_and_b32 s5, s6, 0xffff
; VI-NEXT: s_lshl_b32 s4, s4, 16
; VI-NEXT: s_or_b32 s4, s5, s4
; VI-NEXT: s_sub_i32 s5, s6, s7
; VI-NEXT: s_and_b32 s4, s4, 0xffff
; VI-NEXT: s_lshl_b32 s5, s5, 16
; VI-NEXT: s_or_b32 s4, s4, s5
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
@ -614,12 +614,12 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(<2 x i64> addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: global_load_dword v1, v[2:3], off
; GFX9-NEXT: s_mov_b32 s0, s4
; GFX9-NEXT: s_mov_b32 s1, s5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_sub_i16 v1, v0, v2
; GFX9-NEXT: v_pk_sub_i16 v1, v0, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX9-NEXT: v_bfe_i32 v0, v1, 0, 16
; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 16
@ -642,15 +642,15 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i64(<2 x i64> addrspace(1)
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; VI-NEXT: flat_load_dword v2, v[2:3]
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: flat_load_dword v1, v[2:3]
; VI-NEXT: s_mov_b32 s0, s4
; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: v_sub_u16_sdwa v1, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_sub_u16_e32 v0, v0, v2
; VI-NEXT: v_bfe_i32 v2, v1, 0, 16
; VI-NEXT: v_sub_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-NEXT: v_sub_u16_e32 v0, v0, v1
; VI-NEXT: v_bfe_i32 v0, v0, 0, 16
; VI-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0

8
test/CodeGen/AMDGPU/trunc-combine.ll
View File

@ -106,13 +106,13 @@ define amdgpu_kernel void @truncate_high_elt_extract_vector(<2 x i16> addrspace(
; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_load_dword s2, s[4:5], 0x0
; VI-NEXT: s_load_dword s3, s[6:7], 0x0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_load_dword s0, s[4:5], 0x0
; VI-NEXT: s_load_dword s1, s[6:7], 0x0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_sext_i32_i16 s0, s2
; VI-NEXT: s_sext_i32_i16 s1, s3
; VI-NEXT: s_sext_i32_i16 s0, s0
; VI-NEXT: s_sext_i32_i16 s1, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_mul_i32_i24_e32 v2, s1, v2
; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2

178
test/CodeGen/AMDGPU/vector_shuffle.packed.ll
View File

@ -19,11 +19,11 @@ define <4 x half> @shuffle_v4f16_234u(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_234u:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -114,15 +114,15 @@ define <4 x half> @shuffle_v4f16_35u5(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_35u5:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
; GFX9-NEXT: global_load_dword v1, v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -134,15 +134,15 @@ define <4 x half> @shuffle_v4f16_357u(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_357u:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -181,10 +181,10 @@ define <4 x half> @shuffle_v4f16_0145(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_0145:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -196,10 +196,11 @@ define <4 x half> @shuffle_v4f16_0167(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_0167:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -239,11 +240,11 @@ define <4 x half> @shuffle_v4f16_2345(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_2345:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -255,11 +256,10 @@ define <4 x half> @shuffle_v4f16_2367(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_2367:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: v_mov_b32_e32 v0, v5
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -271,11 +271,10 @@ define <4 x half> @shuffle_v4f16_4501(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_4501:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[3:4], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -287,8 +286,8 @@ define <4 x half> @shuffle_v4f16_4523(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_4523:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
@ -329,11 +328,11 @@ define <4 x half> @shuffle_v4f16_6701(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_6701:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v1
; GFX9-NEXT: v_mov_b32_e32 v1, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -345,8 +344,8 @@ define <4 x half> @shuffle_v4f16_6723(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_6723:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
@ -390,9 +389,9 @@ define <4 x half> @shuffle_v4f16_2356(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v0, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v5
@ -409,9 +408,9 @@ define <4 x half> @shuffle_v4f16_5623(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v0, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
@ -424,15 +423,15 @@ define <4 x half> @shuffle_v4f16_3456(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_3456:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -444,15 +443,15 @@ define <4 x half> @shuffle_v4f16_5634(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_5634:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v3
; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -464,16 +463,16 @@ define <4 x half> @shuffle_v4f16_5734(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_5734:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NEXT: v_and_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v3
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v2
; GFX9-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v3, 16, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1
@ -487,9 +486,9 @@ define <4 x i16> @shuffle_v4i16_2356(<4 x i16> addrspace(1)* %arg0, <4 x i16> ad
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v0, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, v5
@ -504,10 +503,11 @@ define <4 x i16> @shuffle_v4i16_0167(<4 x i16> addrspace(1)* %arg0, <4 x i16> ad
; GFX9-LABEL: shuffle_v4i16_0167:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x i16>, <4 x i16> addrspace(1)* %arg0
%val1 = load <4 x i16>, <4 x i16> addrspace(1)* %arg1
@ -571,12 +571,12 @@ define <4 x half> @shuffle_v4f16_6161(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_6161:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4
; GFX9-NEXT: global_load_dword v0, v[0:1], off
; GFX9-NEXT: global_load_dword v1, v[2:3], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-NEXT: v_mov_b32_e32 v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
@ -651,10 +651,8 @@ define <4 x half> @shuffle_v8f16_4589(<8 x half> addrspace(1)* %arg0, <8 x half>
; GFX9-LABEL: shuffle_v8f16_4589:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v2, v[2:3], off
; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:8
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v1, v2
; GFX9-NEXT: global_load_dword v1, v[2:3], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
@ -667,10 +665,8 @@ define <4 x half> @shuffle_v8f16_10_11_2_3(<8 x half> addrspace(1)* %arg0, <8 x
; GFX9-LABEL: shuffle_v8f16_10_11_2_3:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v2, v[2:3], off offset:4
; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <8 x half>, <8 x half> addrspace(1)* %arg0
@ -685,9 +681,9 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(<8 x half> addrspace(1)* %arg0, <8 x
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx4 v[2:5], v[2:3], off
; GFX9-NEXT: global_load_dword v1, v[0:1], off offset:4
; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_and_b32_sdwa v0, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v5, 16, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@ -732,10 +728,13 @@ define <6 x half> @shuffle_v6f16_452367(<6 x half> addrspace(1)* %arg0, <6 x hal
; GFX9-LABEL: shuffle_v6f16_452367:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v4, v3
; GFX9-NEXT: v_mov_b32_e32 v3, v2
; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_load_dword v3, v[3:4], off
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x half>, <6 x half> addrspace(1)* %arg0
@ -760,9 +759,9 @@ define amdgpu_kernel void @fma_shuffle(<4 x half> addrspace(1)* nocapture readon
; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s4, v4
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX9-NEXT: global_load_dwordx2 v[6:7], v[4:5], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[6:7], v[4:5], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_fma_f16 v6, v0, v2, v6 op_sel_hi:[0,1,1]
; GFX9-NEXT: v_pk_fma_f16 v2, v1, v2, v7 op_sel_hi:[0,1,1]
@ -803,14 +802,15 @@ define <4 x half> @shuffle_v4f16_0456(<4 x half> addrspace(1)* %arg0, <4 x half>
; GFX9-LABEL: shuffle_v4f16_0456:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: v_and_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_b32_e32 v0, v1, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v4
; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-NEXT: v_and_b32_e32 v0, v3, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, <4 x half> addrspace(1)* %arg0
%val1 = load <4 x half>, <4 x half> addrspace(1)* %arg1

2
test/CodeGen/AMDGPU/wait.ll
View File

@ -13,7 +13,7 @@
; DEFAULT: buffer_load_format_xyzw
; DEFAULT: s_waitcnt vmcnt(0)
; DEFAULT: exp
; DEFAULT-NEXT: exp
; DEFAULT: exp
; DEFAULT-NEXT: s_endpgm
define amdgpu_vs void @main(<16 x i8> addrspace(4)* inreg %arg, <16 x i8> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, <16 x i8> addrspace(4)* inreg %arg3, <16 x i8> addrspace(4)* inreg %arg4, i32 inreg %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(4)* inreg %constptr) #0 {
main_body: