mirror of
https://github.com/RPCS3/llvm.git
synced 2024-12-13 14:47:00 +00:00
AMDGPU: Fold clamp modifier for packed instructions
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@312297 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Matt Arsenault
parent
3ccf5513f6
commit
fcd77e8a04
@ -67,11 +67,22 @@ enum : uint64_t {
|
||||
SCALAR_STORE = UINT64_C(1) << 39,
|
||||
FIXED_SIZE = UINT64_C(1) << 40,
|
||||
VOPAsmPrefer32Bit = UINT64_C(1) << 41,
|
||||
HasFPClamp = UINT64_C(1) << 42,
|
||||
VOP3_OPSEL = UINT64_C(1) << 43,
|
||||
maybeAtomic = UINT64_C(1) << 44,
|
||||
F16_ZFILL = UINT64_C(1) << 45,
|
||||
IntClamp = UINT64_C(1) << 46
|
||||
VOP3_OPSEL = UINT64_C(1) << 42,
|
||||
maybeAtomic = UINT64_C(1) << 43,
|
||||
F16_ZFILL = UINT64_C(1) << 44,
|
||||
|
||||
// Is a clamp on FP type.
|
||||
FPClamp = UINT64_C(1) << 45,
|
||||
|
||||
// Is an integer clamp
|
||||
IntClamp = UINT64_C(1) << 46,
|
||||
|
||||
// Clamps lo component of register.
|
||||
ClampLo = UINT64_C(1) << 47,
|
||||
|
||||
// Clamps hi component of register.
|
||||
// ClampLo and ClampHi set for packed clamp.
|
||||
ClampHi = UINT64_C(1) << 48
|
||||
};
|
||||
|
||||
// v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
|
||||
|
||||
@ -728,7 +728,8 @@ const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const {
|
||||
switch (Op) {
|
||||
case AMDGPU::V_MAX_F32_e64:
|
||||
case AMDGPU::V_MAX_F16_e64:
|
||||
case AMDGPU::V_MAX_F64: {
|
||||
case AMDGPU::V_MAX_F64:
|
||||
case AMDGPU::V_PK_MAX_F16: {
|
||||
if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
|
||||
return nullptr;
|
||||
|
||||
@ -741,9 +742,18 @@ const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const {
|
||||
return nullptr;
|
||||
|
||||
// Can't fold up if we have modifiers.
|
||||
if (TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
|
||||
TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
|
||||
TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
|
||||
if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
|
||||
return nullptr;
|
||||
|
||||
unsigned Src0Mods
|
||||
= TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm();
|
||||
unsigned Src1Mods
|
||||
= TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
|
||||
|
||||
// Having a 0 op_sel_hi would require swizzling the output in the source
|
||||
// instruction, which we can't do.
|
||||
unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 : 0;
|
||||
if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
|
||||
return nullptr;
|
||||
return Src0;
|
||||
}
|
||||
@ -771,8 +781,11 @@ bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) {
|
||||
return false;
|
||||
|
||||
MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg());
|
||||
if (!TII->hasFPClamp(*Def))
|
||||
|
||||
// The type of clamp must be compatible.
|
||||
if (TII->getClampMask(*Def) != TII->getClampMask(MI))
|
||||
return false;
|
||||
|
||||
MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
|
||||
if (!DefClamp)
|
||||
return false;
|
||||
|
||||
@ -79,10 +79,6 @@ class InstSI <dag outs, dag ins, string asm = "",
|
||||
// is unable to infer the encoding from the operands.
|
||||
field bit VOPAsmPrefer32Bit = 0;
|
||||
|
||||
// This bit indicates that this has a floating point result type, so
|
||||
// the clamp modifier has floating point semantics.
|
||||
field bit FPClamp = 0;
|
||||
|
||||
// This bit indicates that this is a VOP3 opcode which supports op_sel
|
||||
// modifier (gfx9 only).
|
||||
field bit VOP3_OPSEL = 0;
|
||||
@ -94,10 +90,22 @@ class InstSI <dag outs, dag ins, string asm = "",
|
||||
// unused bits in dst. Note that new GFX9 opcodes preserve unused bits.
|
||||
field bit F16_ZFILL = 0;
|
||||
|
||||
// This bit indicates that this has a floating point result type, so
|
||||
// the clamp modifier has floating point semantics.
|
||||
field bit FPClamp = 0;
|
||||
|
||||
// This bit indicates that instruction may support integer clamping
|
||||
// which depends on GPU features.
|
||||
field bit IntClamp = 0;
|
||||
|
||||
// This field indicates that the clamp applies to the low component
|
||||
// of a packed output register.
|
||||
field bit ClampLo = 0;
|
||||
|
||||
// This field indicates that the clamp applies to the high component
|
||||
// of a packed output register.
|
||||
field bit ClampHi = 0;
|
||||
|
||||
// These need to be kept in sync with the enum in SIInstrFlags.
|
||||
let TSFlags{0} = SALU;
|
||||
let TSFlags{1} = VALU;
|
||||
@ -141,12 +149,15 @@ class InstSI <dag outs, dag ins, string asm = "",
|
||||
let TSFlags{39} = ScalarStore;
|
||||
let TSFlags{40} = FixedSize;
|
||||
let TSFlags{41} = VOPAsmPrefer32Bit;
|
||||
let TSFlags{42} = FPClamp;
|
||||
let TSFlags{43} = VOP3_OPSEL;
|
||||
let TSFlags{42} = VOP3_OPSEL;
|
||||
|
||||
let TSFlags{44} = maybeAtomic;
|
||||
let TSFlags{45} = F16_ZFILL;
|
||||
let TSFlags{43} = maybeAtomic;
|
||||
let TSFlags{44} = F16_ZFILL;
|
||||
|
||||
let TSFlags{45} = FPClamp;
|
||||
let TSFlags{46} = IntClamp;
|
||||
let TSFlags{47} = ClampLo;
|
||||
let TSFlags{48} = ClampHi;
|
||||
|
||||
let SchedRW = [Write32Bit];
|
||||
|
||||
|
||||
@ -548,11 +548,23 @@ public:
|
||||
}
|
||||
|
||||
static bool hasFPClamp(const MachineInstr &MI) {
|
||||
return MI.getDesc().TSFlags & SIInstrFlags::HasFPClamp;
|
||||
return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
|
||||
}
|
||||
|
||||
bool hasFPClamp(uint16_t Opcode) const {
|
||||
return get(Opcode).TSFlags & SIInstrFlags::HasFPClamp;
|
||||
return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
|
||||
}
|
||||
|
||||
static bool hasIntClamp(const MachineInstr &MI) {
|
||||
return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
|
||||
}
|
||||
|
||||
uint64_t getClampMask(const MachineInstr &MI) const {
|
||||
const uint64_t ClampFlags = SIInstrFlags::FPClamp |
|
||||
SIInstrFlags::IntClamp |
|
||||
SIInstrFlags::ClampLo |
|
||||
SIInstrFlags::ClampHi;
|
||||
return MI.getDesc().TSFlags & ClampFlags;
|
||||
}
|
||||
|
||||
bool isVGPRCopy(const MachineInstr &MI) const {
|
||||
|
||||
@ -1529,6 +1529,8 @@ class VOPProfile <list<ValueType> _ArgVT> {
|
||||
field bit HasSDWAClamp = EmitDst;
|
||||
field bit HasFPClamp = BitAnd<isFloatType<DstVT>.ret, HasClamp>.ret;
|
||||
field bit HasIntClamp = !if(isFloatType<DstVT>.ret, 0, HasClamp);
|
||||
field bit HasClampLo = HasClamp;
|
||||
field bit HasClampHi = BitAnd<isPackedType<DstVT>.ret, HasClamp>.ret;
|
||||
field bit HasHigh = 0;
|
||||
|
||||
field bit IsPacked = isPackedType<Src0VT>.ret;
|
||||
|
||||
@ -106,6 +106,10 @@ class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern = [],
|
||||
let VOP3 = 1;
|
||||
let VALU = 1;
|
||||
let FPClamp = P.HasFPClamp;
|
||||
let IntClamp = P.HasIntClamp;
|
||||
let ClampLo = P.HasClampLo;
|
||||
let ClampHi = P.HasClampHi;
|
||||
|
||||
let Uses = [EXEC];
|
||||
|
||||
let AsmVariantName = AMDGPUAsmVariants.VOP3;
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
|
||||
; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
|
||||
; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
|
||||
; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89 %s
|
||||
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_f32:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN-NOT: [[A]]
|
||||
; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
|
||||
@ -18,7 +19,7 @@ define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float a
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_multi_use_src_f32:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
|
||||
; GCN: v_max_f32_e64 v{{[0-9]+}}, [[ADD]], [[ADD]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
|
||||
@ -35,7 +36,7 @@ define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, f
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_dbg_use_src_f32:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN-NOT: [[A]]
|
||||
; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
|
||||
@ -52,7 +53,7 @@ define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, flo
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_neg_src_f32:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: v_floor_f32_e32 [[FLOOR:v[0-9]+]], [[A]]
|
||||
; GCN: v_max_f32_e64 v{{[0-9]+}}, -[[FLOOR]], -[[FLOOR]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
|
||||
@ -69,7 +70,7 @@ define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, flo
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_non_clamp_max_f32:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
|
||||
; GCN: v_max_f32_e32 v{{[0-9]+}}, 0, [[ADD]]{{$}}
|
||||
define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
|
||||
@ -84,7 +85,7 @@ define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float a
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_f32_denormals:
|
||||
; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GCN: v_add_f32_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
@ -99,8 +100,8 @@ define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %ou
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_f16_denorm:
|
||||
; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
|
||||
; VI: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
|
||||
; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
|
||||
; GFX89: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
|
||||
|
||||
; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
|
||||
; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
|
||||
@ -118,9 +119,9 @@ define amdgpu_kernel void @v_clamp_add_src_f16_denorm(half addrspace(1)* %out, h
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_f16_no_denormals:
|
||||
; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
|
||||
; VI-NOT: [[A]]
|
||||
; VI: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
|
||||
; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
|
||||
; GFX89-NOT: [[A]]
|
||||
; GFX89: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
|
||||
|
||||
; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
|
||||
; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
|
||||
@ -138,7 +139,7 @@ define amdgpu_kernel void @v_clamp_add_src_f16_no_denormals(half addrspace(1)* %
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f32:
|
||||
; GCN: {{buffer|flat}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
|
||||
; GCN: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
|
||||
; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[A]], 1.0 clamp{{$}}
|
||||
; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[B]], 1.0 clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %aptr) #0 {
|
||||
@ -154,7 +155,7 @@ define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out,
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_f64:
|
||||
; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
|
||||
; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
|
||||
; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], 1.0 clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
@ -185,6 +186,173 @@ define amdgpu_kernel void @v_clamp_mac_to_mad(float addrspace(1)* %out, float ad
|
||||
ret void
|
||||
}
|
||||
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_no_denormals:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_no_denormals(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #3 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%neg.add = fsub <2 x half> <half -0.0, half -0.0>, %add
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.add, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_lo:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_lo(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%lo = extractelement <2 x half> %add, i32 0
|
||||
%neg.lo = fsub half -0.0, %lo
|
||||
%neg.lo.add = insertelement <2 x half> %add, half %neg.lo, i32 0
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.lo.add, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_hi:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_hi:[1,1] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_hi(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%hi = extractelement <2 x half> %add, i32 1
|
||||
%neg.hi = fsub half -0.0, %hi
|
||||
%neg.hi.add = insertelement <2 x half> %add, half %neg.hi, i32 1
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.hi.add, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_shuf:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] op_sel:[1,1] op_sel_hi:[0,0] clamp{{$}}
|
||||
define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_shuf(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%shuf = shufflevector <2 x half> %add, <2 x half> undef, <2 x i32> <i32 1, i32 0>
|
||||
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f32_src:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f32_src(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%bc = bitcast <2 x half> %a to float
|
||||
%f32.op = fadd float %bc, 1.0
|
||||
%f32.op.cast = bitcast float %f32.op to <2 x half>
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %f32.op.cast, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; GCN-LABEL: {{^}}v_no_clamp_add_packed_src_f32:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
|
||||
; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
|
||||
; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
|
||||
; GFX9: v_max_f32_e64 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_no_clamp_add_packed_src_f32(float addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
|
||||
%a = load <2 x half>, <2 x half> addrspace(1)* %gep0
|
||||
%add = fadd <2 x half> %a, <half 1.0, half 1.0>
|
||||
%bc.add = bitcast <2 x half> %add to float
|
||||
%max = call float @llvm.maxnum.f32(float %bc.add, float 0.0)
|
||||
%clamp = call float @llvm.minnum.f32(float %max, float 1.0)
|
||||
store float %clamp, float addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
; Since the high bits are zeroed, it probably would be OK in this case
|
||||
; to use clamp.
|
||||
; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f16_src:
|
||||
; GCN-DAG: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
|
||||
; GFX9: v_add_f16_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
|
||||
; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
|
||||
define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(<2 x half> addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
|
||||
%tid = call i32 @llvm.amdgcn.workitem.id.x()
|
||||
%gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
|
||||
%out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
|
||||
%a = load half, half addrspace(1)* %gep0
|
||||
%add = fadd half %a, 1.0
|
||||
%bc = bitcast half %add to i16
|
||||
%zext = zext i16 %bc to i32
|
||||
%v2f16 = bitcast i32 %zext to <2 x half>
|
||||
%max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %v2f16, <2 x half> zeroinitializer)
|
||||
%clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
|
||||
store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
|
||||
ret void
|
||||
}
|
||||
|
||||
declare i32 @llvm.amdgcn.workitem.id.x() #1
|
||||
declare float @llvm.fabs.f32(float) #1
|
||||
declare float @llvm.floor.f32(float) #1
|
||||
@ -197,8 +365,12 @@ declare double @llvm.maxnum.f64(double, double) #1
|
||||
declare half @llvm.fabs.f16(half) #1
|
||||
declare half @llvm.minnum.f16(half, half) #1
|
||||
declare half @llvm.maxnum.f16(half, half) #1
|
||||
declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
|
||||
declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
|
||||
declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1
|
||||
declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1
|
||||
|
||||
|
||||
declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
|
||||
|
||||
attributes #0 = { nounwind }
|
||||
|
||||
Loading…
Reference in New Issue
Block a user