diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp index 1d3bf224cda..09779bef057 100644 --- a/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/lib/Target/PowerPC/PPCISelLowering.cpp @@ -206,6 +206,11 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM) setOperationAction(ISD::ATOMIC_LAS , MVT::i32 , Custom); setOperationAction(ISD::ATOMIC_LCS , MVT::i32 , Custom); setOperationAction(ISD::ATOMIC_SWAP , MVT::i32 , Custom); + if (TM.getSubtarget().has64BitSupport()) { + setOperationAction(ISD::ATOMIC_LAS , MVT::i64 , Custom); + setOperationAction(ISD::ATOMIC_LCS , MVT::i64 , Custom); + setOperationAction(ISD::ATOMIC_SWAP , MVT::i64 , Custom); + } // We want to custom lower some of our intrinsics. setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); @@ -398,8 +403,8 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { case PPCISD::VCMPo: return "PPCISD::VCMPo"; case PPCISD::LBRX: return "PPCISD::LBRX"; case PPCISD::STBRX: return "PPCISD::STBRX"; - case PPCISD::LWARX: return "PPCISD::LWARX"; - case PPCISD::STWCX: return "PPCISD::STWCX"; + case PPCISD::LARX: return "PPCISD::LARX"; + case PPCISD::STCX: return "PPCISD::STCX"; case PPCISD::CMP_UNRESERVE: return "PPCISD::CMP_UNRESERVE"; case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; case PPCISD::MFFS: return "PPCISD::MFFS"; @@ -2304,7 +2309,7 @@ SDOperand PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op, } SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) { - MVT::ValueType VT = Op.getValueType(); + MVT::ValueType VT = Op.Val->getValueType(0); SDOperand Chain = Op.getOperand(0); SDOperand Ptr = Op.getOperand(1); SDOperand Incr = Op.getOperand(2); @@ -2316,11 +2321,11 @@ SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) { SDOperand Label = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32); SDOperand Ops[] = { - Chain, // Chain - Ptr, // Ptr - Label, // Label + Chain, // Chain + Ptr, // Ptr + Label, // Label }; - SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3); + SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3); Chain = Load.getValue(1); // Compute new value. @@ -2328,19 +2333,19 @@ SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) { // Issue a "store and check". SDOperand Ops2[] = { - Chain, // Chain - NewVal, // Value - Ptr, // Ptr - Label, // Label + Chain, // Chain + NewVal, // Value + Ptr, // Ptr + Label, // Label }; - SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4); + SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops2, 4); SDOperand OutOps[] = { Load, Store }; return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), OutOps, 2); } SDOperand PPCTargetLowering::LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG) { - MVT::ValueType VT = Op.getValueType(); + MVT::ValueType VT = Op.Val->getValueType(0); SDOperand Chain = Op.getOperand(0); SDOperand Ptr = Op.getOperand(1); SDOperand NewVal = Op.getOperand(2); @@ -2353,37 +2358,37 @@ SDOperand PPCTargetLowering::LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG) { SDOperand Label = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32); SDOperand Ops[] = { - Chain, // Chain - Ptr, // Ptr - Label, // Label + Chain, // Chain + Ptr, // Ptr + Label, // Label }; - SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3); + SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3); Chain = Load.getValue(1); // Compare and unreserve if not equal. SDOperand Ops2[] = { - Chain, // Chain - OldVal, // Old value - Load, // Value in memory - Label, // Label + Chain, // Chain + OldVal, // Old value + Load, // Value in memory + Label, // Label }; Chain = DAG.getNode(PPCISD::CMP_UNRESERVE, MVT::Other, Ops2, 4); // Issue a "store and check". SDOperand Ops3[] = { - Chain, // Chain - NewVal, // Value - Ptr, // Ptr - Label, // Label + Chain, // Chain + NewVal, // Value + Ptr, // Ptr + Label, // Label }; - SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops3, 4); + SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops3, 4); SDOperand OutOps[] = { Load, Store }; return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), OutOps, 2); } SDOperand PPCTargetLowering::LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG) { - MVT::ValueType VT = Op.getValueType(); + MVT::ValueType VT = Op.Val->getValueType(0); SDOperand Chain = Op.getOperand(0); SDOperand Ptr = Op.getOperand(1); SDOperand NewVal = Op.getOperand(2); @@ -2395,21 +2400,21 @@ SDOperand PPCTargetLowering::LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG) { SDOperand Label = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32); SDOperand Ops[] = { - Chain, // Chain - Ptr, // Ptr - Label, // Label + Chain, // Chain + Ptr, // Ptr + Label, // Label }; - SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3); + SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3); Chain = Load.getValue(1); // Issue a "store and check". SDOperand Ops2[] = { - Chain, // Chain - NewVal, // Value - Ptr, // Ptr - Label, // Label + Chain, // Chain + NewVal, // Value + Ptr, // Ptr + Label, // Label }; - SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4); + SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops2, 4); SDOperand OutOps[] = { Load, Store }; return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), OutOps, 2); diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h index d9ced1b3366..bd13baf24ac 100644 --- a/lib/Target/PowerPC/PPCISelLowering.h +++ b/lib/Target/PowerPC/PPCISelLowering.h @@ -152,13 +152,13 @@ namespace llvm { /// MTFSF = F8RC, INFLAG - This moves the register into the FPSCR. MTFSF, - /// LWARX = This corresponds to PPC lwarx instrcution: load word and + /// LARX = This corresponds to PPC l{w|d}arx instrcution: load and /// reserve indexed. This is used to implement atomic operations. - LWARX, + LARX, - /// STWCX = This corresponds to PPC stwcx. instrcution: store word - /// conditional indexed. This is used to implement atomic operations. - STWCX, + /// STCX = This corresponds to PPC stcx. instrcution: store conditional + /// indexed. This is used to implement atomic operations. + STCX, /// CMP_UNRESERVE = Test for equality and "unreserve" if not true. This /// is used to implement atomic operations. diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td index 0b7d1cc42a7..0b23c67f94b 100644 --- a/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/lib/Target/PowerPC/PPCInstr64Bit.td @@ -116,6 +116,25 @@ def : Pat<(PPCcall_ELF (i64 tglobaladdr:$dst)), def : Pat<(PPCcall_ELF (i64 texternalsym:$dst)), (BL8_ELF texternalsym:$dst)>; + +// Atomic operations. +def LDARX : Pseudo<(outs G8RC:$rD), (ins memrr:$ptr, i32imm:$label), + "\nLa${label}_entry:\n\tldarx $rD, $ptr", + [(set G8RC:$rD, (PPClarx xoaddr:$ptr, imm:$label))]>; + +let Defs = [CR0] in { +def STDCX : Pseudo<(outs), (ins G8RC:$rS, memrr:$dst, i32imm:$label), + "stdcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:", + [(PPCstcx G8RC:$rS, xoaddr:$dst, imm:$label)]>; + +def CMP_UNRESd : Pseudo<(outs), (ins G8RC:$rA, G8RC:$rB, i32imm:$label), + "cmpd $rA, $rB\n\tbne- La${label}_exit", + [(PPCcmp_unres G8RC:$rA, G8RC:$rB, imm:$label)]>; +def CMP_UNRESdi : Pseudo<(outs), (ins G8RC:$rA, s16imm64:$imm, i32imm:$label), + "cmpdi $rA, $imm\n\tbne- La${label}_exit", + [(PPCcmp_unres G8RC:$rA, immSExt16:$imm, imm:$label)]>; +} + //===----------------------------------------------------------------------===// // 64-bit SPR manipulation instrs. diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td index 7cbdac3c1c1..a765494eec8 100644 --- a/lib/Target/PowerPC/PPCInstrInfo.td +++ b/lib/Target/PowerPC/PPCInstrInfo.td @@ -42,14 +42,14 @@ def SDT_PPCstbrx : SDTypeProfile<0, 4, [ SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT> ]>; -def SDT_PPClwarx : SDTypeProfile<1, 2, [ - SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32> +def SDT_PPClarx : SDTypeProfile<1, 2, [ + SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32> ]>; -def SDT_PPCstwcx : SDTypeProfile<0, 3, [ - SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32> +def SDT_PPCstcx : SDTypeProfile<0, 3, [ + SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32> ]>; def SDT_PPCcmp_unres : SDTypeProfile<0, 3, [ - SDTCisVT<0, i32>, SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2> + SDTCisSameAs<0, 1>, SDTCisInt<1>, SDTCisVT<2, i32> ]>; //===----------------------------------------------------------------------===// @@ -132,10 +132,10 @@ def PPClbrx : SDNode<"PPCISD::LBRX", SDT_PPClbrx, def PPCstbrx : SDNode<"PPCISD::STBRX", SDT_PPCstbrx, [SDNPHasChain, SDNPMayStore]>; -def PPClwarx : SDNode<"PPCISD::LWARX", SDT_PPClwarx, - [SDNPHasChain, SDNPMayLoad]>; -def PPCstwcx : SDNode<"PPCISD::STWCX", SDT_PPCstwcx, - [SDNPHasChain, SDNPMayStore]>; +def PPClarx : SDNode<"PPCISD::LARX", SDT_PPClarx, + [SDNPHasChain, SDNPMayLoad]>; +def PPCstcx : SDNode<"PPCISD::STCX", SDT_PPCstcx, + [SDNPHasChain, SDNPMayStore]>; def PPCcmp_unres : SDNode<"PPCISD::CMP_UNRESERVE", SDT_PPCcmp_unres, [SDNPHasChain]>; @@ -482,19 +482,19 @@ def DCBZL : DCB_Form<1014, 1, (outs), (ins memrr:$dst), // Atomic operations. def LWARX : Pseudo<(outs GPRC:$rD), (ins memrr:$ptr, i32imm:$label), "\nLa${label}_entry:\n\tlwarx $rD, $ptr", - [(set GPRC:$rD, (PPClwarx xoaddr:$ptr, imm:$label))]>; + [(set GPRC:$rD, (PPClarx xoaddr:$ptr, imm:$label))]>; let Defs = [CR0] in { def STWCX : Pseudo<(outs), (ins GPRC:$rS, memrr:$dst, i32imm:$label), - "stwcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:", - [(PPCstwcx GPRC:$rS, xoaddr:$dst, imm:$label)]>; + "stwcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:", + [(PPCstcx GPRC:$rS, xoaddr:$dst, imm:$label)]>; def CMP_UNRESw : Pseudo<(outs), (ins GPRC:$rA, GPRC:$rB, i32imm:$label), "cmpw $rA, $rB\n\tbne- La${label}_exit", [(PPCcmp_unres GPRC:$rA, GPRC:$rB, imm:$label)]>; def CMP_UNRESwi : Pseudo<(outs), (ins GPRC:$rA, s16imm:$imm, i32imm:$label), "cmpwi $rA, $imm\n\tbne- La${label}_exit", - [(PPCcmp_unres GPRC:$rA, imm:$imm, imm:$label)]>; + [(PPCcmp_unres GPRC:$rA, immSExt16:$imm, imm:$label)]>; } //===----------------------------------------------------------------------===// @@ -1265,8 +1265,8 @@ def : Pat<(extloadf32 xaddr:$src), (FMRSD (LFSX xaddr:$src))>; // Atomic operations -def : Pat<(PPCcmp_unres imm:$imm, GPRC:$rA, imm:$label), - (CMP_UNRESwi GPRC:$rA, imm:$imm, imm:$label)>; +def : Pat<(PPCcmp_unres immSExt16:$imm, GPRC:$rA, imm:$label), + (CMP_UNRESwi GPRC:$rA, immSExt16:$imm, imm:$label)>; include "PPCInstrAltivec.td" include "PPCInstr64Bit.td" diff --git a/test/CodeGen/PowerPC/atomic-1.ll b/test/CodeGen/PowerPC/atomic-1.ll new file mode 100644 index 00000000000..74b9c6a4f6e --- /dev/null +++ b/test/CodeGen/PowerPC/atomic-1.ll @@ -0,0 +1,27 @@ +; RUN: llvm-as < %s | llc -march=ppc32 | grep lwarx | count 4 +; RUN: llvm-as < %s | llc -march=ppc32 | grep stwcx. | count 4 + +define i32 @exchange_and_add(i32* %mem, i32 %val) nounwind { + %tmp = call i32 @llvm.atomic.las.i32( i32* %mem, i32 %val ) + ret i32 %tmp +} + +define i32 @exchange_and_cmp(i32* %mem) nounwind { + %tmp = call i32 @llvm.atomic.lcs.i32( i32* %mem, i32 0, i32 1 ) + ret i32 %tmp +} + +define i16 @exchange_and_cmp16(i16* %mem) nounwind { + %tmp = call i16 @llvm.atomic.lcs.i16( i16* %mem, i16 0, i16 1 ) + ret i16 %tmp +} + +define i32 @exchange(i32* %mem, i32 %val) nounwind { + %tmp = call i32 @llvm.atomic.swap.i32( i32* %mem, i32 1 ) + ret i32 %tmp +} + +declare i32 @llvm.atomic.las.i32(i32*, i32) nounwind +declare i32 @llvm.atomic.lcs.i32(i32*, i32, i32) nounwind +declare i16 @llvm.atomic.lcs.i16(i16*, i16, i16) nounwind +declare i32 @llvm.atomic.swap.i32(i32*, i32) nounwind diff --git a/test/CodeGen/PowerPC/atomic-2.ll b/test/CodeGen/PowerPC/atomic-2.ll new file mode 100644 index 00000000000..8e1ef1b0f87 --- /dev/null +++ b/test/CodeGen/PowerPC/atomic-2.ll @@ -0,0 +1,21 @@ +; RUN: llvm-as < %s | llc -march=ppc64 | grep ldarx | count 3 +; RUN: llvm-as < %s | llc -march=ppc64 | grep stdcx. | count 3 + +define i64 @exchange_and_add(i64* %mem, i64 %val) nounwind { + %tmp = call i64 @llvm.atomic.las.i64( i64* %mem, i64 %val ) + ret i64 %tmp +} + +define i64 @exchange_and_cmp(i64* %mem) nounwind { + %tmp = call i64 @llvm.atomic.lcs.i64( i64* %mem, i64 0, i64 1 ) + ret i64 %tmp +} + +define i64 @exchange(i64* %mem, i64 %val) nounwind { + %tmp = call i64 @llvm.atomic.swap.i64( i64* %mem, i64 1 ) + ret i64 %tmp +} + +declare i64 @llvm.atomic.las.i64(i64*, i64) nounwind +declare i64 @llvm.atomic.lcs.i64(i64*, i64, i64) nounwind +declare i64 @llvm.atomic.swap.i64(i64*, i64) nounwind