2013-01-11 20:05:37 +00:00
|
|
|
//===-- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ---===//
|
|
|
|
|
//
|
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
|
//
|
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
//
|
|
|
|
|
// This implements the TargetLoweringBase class.
|
|
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
#include "llvm/ADT/BitVector.h"
|
|
|
|
|
#include "llvm/ADT/STLExtras.h"
|
2016-10-20 16:55:45 +00:00
|
|
|
#include "llvm/ADT/StringExtras.h"
|
2013-02-15 18:45:18 +00:00
|
|
|
#include "llvm/ADT/Triple.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/CodeGen/Analysis.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineFrameInfo.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
2013-11-29 03:07:54 +00:00
|
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/CodeGen/MachineJumpTableInfo.h"
|
2017-04-28 20:25:05 +00:00
|
|
|
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
2013-11-29 03:07:54 +00:00
|
|
|
#include "llvm/CodeGen/StackMaps.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/IR/DataLayout.h"
|
|
|
|
|
#include "llvm/IR/DerivedTypes.h"
|
|
|
|
|
#include "llvm/IR/GlobalVariable.h"
|
2014-02-19 17:23:20 +00:00
|
|
|
#include "llvm/IR/Mangler.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/MC/MCAsmInfo.h"
|
2014-02-19 17:23:20 +00:00
|
|
|
#include "llvm/MC/MCContext.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/MC/MCExpr.h"
|
2016-04-26 17:11:17 +00:00
|
|
|
#include "llvm/Support/BranchProbability.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
|
|
|
|
#include "llvm/Support/ErrorHandling.h"
|
|
|
|
|
#include "llvm/Support/MathExtras.h"
|
2017-06-06 11:49:48 +00:00
|
|
|
#include "llvm/Target/TargetLowering.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include "llvm/Target/TargetLoweringObjectFile.h"
|
|
|
|
|
#include "llvm/Target/TargetMachine.h"
|
|
|
|
|
#include "llvm/Target/TargetRegisterInfo.h"
|
2014-08-04 21:25:23 +00:00
|
|
|
#include "llvm/Target/TargetSubtargetInfo.h"
|
2013-01-11 20:05:37 +00:00
|
|
|
#include <cctype>
|
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
2015-07-01 18:10:20 +00:00
|
|
|
static cl::opt<bool> JumpIsExpensiveOverride(
|
|
|
|
|
"jump-is-expensive", cl::init(false),
|
|
|
|
|
cl::desc("Do not create extra branches to split comparison logic."),
|
|
|
|
|
cl::Hidden);
|
|
|
|
|
|
2016-10-25 19:53:51 +00:00
|
|
|
static cl::opt<unsigned> MinimumJumpTableEntries
|
|
|
|
|
("min-jump-table-entries", cl::init(4), cl::Hidden,
|
|
|
|
|
cl::desc("Set minimum number of entries to use a jump table."));
|
|
|
|
|
|
2016-09-26 15:32:33 +00:00
|
|
|
static cl::opt<unsigned> MaximumJumpTableSize
|
2016-10-25 19:53:51 +00:00
|
|
|
("max-jump-table-size", cl::init(0), cl::Hidden,
|
|
|
|
|
cl::desc("Set maximum size of jump tables; zero for no limit."));
|
2016-09-26 15:32:33 +00:00
|
|
|
|
[InlineCost] Improve the cost heuristic for Switch
Summary:
The motivation example is like below which has 13 cases but only 2 distinct targets
```
lor.lhs.false2: ; preds = %if.then
switch i32 %Status, label %if.then27 [
i32 -7012, label %if.end35
i32 -10008, label %if.end35
i32 -10016, label %if.end35
i32 15000, label %if.end35
i32 14013, label %if.end35
i32 10114, label %if.end35
i32 10107, label %if.end35
i32 10105, label %if.end35
i32 10013, label %if.end35
i32 10011, label %if.end35
i32 7008, label %if.end35
i32 7007, label %if.end35
i32 5002, label %if.end35
]
```
which is compiled into a balanced binary tree like this on AArch64 (similar on X86)
```
.LBB853_9: // %lor.lhs.false2
mov w8, #10012
cmp w19, w8
b.gt .LBB853_14
// BB#10: // %lor.lhs.false2
mov w8, #5001
cmp w19, w8
b.gt .LBB853_18
// BB#11: // %lor.lhs.false2
mov w8, #-10016
cmp w19, w8
b.eq .LBB853_23
// BB#12: // %lor.lhs.false2
mov w8, #-10008
cmp w19, w8
b.eq .LBB853_23
// BB#13: // %lor.lhs.false2
mov w8, #-7012
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_14: // %lor.lhs.false2
mov w8, #14012
cmp w19, w8
b.gt .LBB853_21
// BB#15: // %lor.lhs.false2
mov w8, #-10105
add w8, w19, w8
cmp w8, #9 // =9
b.hi .LBB853_17
// BB#16: // %lor.lhs.false2
orr w9, wzr, #0x1
lsl w8, w9, w8
mov w9, #517
and w8, w8, w9
cbnz w8, .LBB853_23
.LBB853_17: // %lor.lhs.false2
mov w8, #10013
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_18: // %lor.lhs.false2
mov w8, #-7007
add w8, w19, w8
cmp w8, #2 // =2
b.lo .LBB853_23
// BB#19: // %lor.lhs.false2
mov w8, #5002
cmp w19, w8
b.eq .LBB853_23
// BB#20: // %lor.lhs.false2
mov w8, #10011
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_21: // %lor.lhs.false2
mov w8, #14013
cmp w19, w8
b.eq .LBB853_23
// BB#22: // %lor.lhs.false2
mov w8, #15000
cmp w19, w8
b.ne .LBB853_3
```
However, the inline cost model estimates the cost to be linear with the number
of distinct targets and the cost of the above switch is just 2 InstrCosts.
The function containing this switch is then inlined about 900 times.
This change use the general way of switch lowering for the inline heuristic. It
etimate the number of case clusters with the suitability check for a jump table
or bit test. Considering the binary search tree built for the clusters, this
change modifies the model to be linear with the size of the balanced binary
tree. The model is off by default for now :
-inline-generic-switch-cost=false
This change was originally proposed by Haicheng in D29870.
Reviewers: hans, bmakam, chandlerc, eraman, haicheng, mcrosier
Reviewed By: hans
Subscribers: joerg, aemerson, llvm-commits, rengolin
Differential Revision: https://reviews.llvm.org/D31085
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@301649 91177308-0d34-0410-b5e6-96231b3b80d8
2017-04-28 16:04:03 +00:00
|
|
|
/// Minimum jump table density for normal functions.
|
|
|
|
|
static cl::opt<unsigned>
|
|
|
|
|
JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
|
|
|
|
|
cl::desc("Minimum density for building a jump table in "
|
|
|
|
|
"a normal function"));
|
|
|
|
|
|
|
|
|
|
/// Minimum jump table density for -Os or -Oz functions.
|
|
|
|
|
static cl::opt<unsigned> OptsizeJumpTableDensity(
|
|
|
|
|
"optsize-jump-table-density", cl::init(40), cl::Hidden,
|
|
|
|
|
cl::desc("Minimum density for building a jump table in "
|
|
|
|
|
"an optsize function"));
|
|
|
|
|
|
2016-04-26 17:11:17 +00:00
|
|
|
// Although this default value is arbitrary, it is not random. It is assumed
|
|
|
|
|
// that a condition that evaluates the same way by a higher percentage than this
|
|
|
|
|
// is best represented as control flow. Therefore, the default value N should be
|
|
|
|
|
// set such that the win from N% correct executions is greater than the loss
|
|
|
|
|
// from (100 - N)% mispredicted executions for the majority of intended targets.
|
|
|
|
|
static cl::opt<int> MinPercentageForPredictableBranch(
|
|
|
|
|
"min-predictable-branch", cl::init(99),
|
|
|
|
|
cl::desc("Minimum percentage (0-100) that a condition must be either true "
|
|
|
|
|
"or false to assume that the condition is predictable"),
|
|
|
|
|
cl::Hidden);
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
/// InitLibcallNames - Set default libcall names.
|
|
|
|
|
///
|
2014-06-02 20:51:49 +00:00
|
|
|
static void InitLibcallNames(const char **Names, const Triple &TT) {
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SHL_I16] = "__ashlhi3";
|
|
|
|
|
Names[RTLIB::SHL_I32] = "__ashlsi3";
|
|
|
|
|
Names[RTLIB::SHL_I64] = "__ashldi3";
|
|
|
|
|
Names[RTLIB::SHL_I128] = "__ashlti3";
|
|
|
|
|
Names[RTLIB::SRL_I16] = "__lshrhi3";
|
|
|
|
|
Names[RTLIB::SRL_I32] = "__lshrsi3";
|
|
|
|
|
Names[RTLIB::SRL_I64] = "__lshrdi3";
|
|
|
|
|
Names[RTLIB::SRL_I128] = "__lshrti3";
|
|
|
|
|
Names[RTLIB::SRA_I16] = "__ashrhi3";
|
|
|
|
|
Names[RTLIB::SRA_I32] = "__ashrsi3";
|
|
|
|
|
Names[RTLIB::SRA_I64] = "__ashrdi3";
|
|
|
|
|
Names[RTLIB::SRA_I128] = "__ashrti3";
|
|
|
|
|
Names[RTLIB::MUL_I8] = "__mulqi3";
|
|
|
|
|
Names[RTLIB::MUL_I16] = "__mulhi3";
|
|
|
|
|
Names[RTLIB::MUL_I32] = "__mulsi3";
|
|
|
|
|
Names[RTLIB::MUL_I64] = "__muldi3";
|
|
|
|
|
Names[RTLIB::MUL_I128] = "__multi3";
|
|
|
|
|
Names[RTLIB::MULO_I32] = "__mulosi4";
|
|
|
|
|
Names[RTLIB::MULO_I64] = "__mulodi4";
|
|
|
|
|
Names[RTLIB::MULO_I128] = "__muloti4";
|
|
|
|
|
Names[RTLIB::SDIV_I8] = "__divqi3";
|
|
|
|
|
Names[RTLIB::SDIV_I16] = "__divhi3";
|
|
|
|
|
Names[RTLIB::SDIV_I32] = "__divsi3";
|
|
|
|
|
Names[RTLIB::SDIV_I64] = "__divdi3";
|
|
|
|
|
Names[RTLIB::SDIV_I128] = "__divti3";
|
|
|
|
|
Names[RTLIB::UDIV_I8] = "__udivqi3";
|
|
|
|
|
Names[RTLIB::UDIV_I16] = "__udivhi3";
|
|
|
|
|
Names[RTLIB::UDIV_I32] = "__udivsi3";
|
|
|
|
|
Names[RTLIB::UDIV_I64] = "__udivdi3";
|
|
|
|
|
Names[RTLIB::UDIV_I128] = "__udivti3";
|
|
|
|
|
Names[RTLIB::SREM_I8] = "__modqi3";
|
|
|
|
|
Names[RTLIB::SREM_I16] = "__modhi3";
|
|
|
|
|
Names[RTLIB::SREM_I32] = "__modsi3";
|
|
|
|
|
Names[RTLIB::SREM_I64] = "__moddi3";
|
|
|
|
|
Names[RTLIB::SREM_I128] = "__modti3";
|
|
|
|
|
Names[RTLIB::UREM_I8] = "__umodqi3";
|
|
|
|
|
Names[RTLIB::UREM_I16] = "__umodhi3";
|
|
|
|
|
Names[RTLIB::UREM_I32] = "__umodsi3";
|
|
|
|
|
Names[RTLIB::UREM_I64] = "__umoddi3";
|
|
|
|
|
Names[RTLIB::UREM_I128] = "__umodti3";
|
|
|
|
|
|
|
|
|
|
Names[RTLIB::NEG_I32] = "__negsi2";
|
|
|
|
|
Names[RTLIB::NEG_I64] = "__negdi2";
|
|
|
|
|
Names[RTLIB::ADD_F32] = "__addsf3";
|
|
|
|
|
Names[RTLIB::ADD_F64] = "__adddf3";
|
|
|
|
|
Names[RTLIB::ADD_F80] = "__addxf3";
|
|
|
|
|
Names[RTLIB::ADD_F128] = "__addtf3";
|
|
|
|
|
Names[RTLIB::ADD_PPCF128] = "__gcc_qadd";
|
|
|
|
|
Names[RTLIB::SUB_F32] = "__subsf3";
|
|
|
|
|
Names[RTLIB::SUB_F64] = "__subdf3";
|
|
|
|
|
Names[RTLIB::SUB_F80] = "__subxf3";
|
|
|
|
|
Names[RTLIB::SUB_F128] = "__subtf3";
|
|
|
|
|
Names[RTLIB::SUB_PPCF128] = "__gcc_qsub";
|
|
|
|
|
Names[RTLIB::MUL_F32] = "__mulsf3";
|
|
|
|
|
Names[RTLIB::MUL_F64] = "__muldf3";
|
|
|
|
|
Names[RTLIB::MUL_F80] = "__mulxf3";
|
|
|
|
|
Names[RTLIB::MUL_F128] = "__multf3";
|
|
|
|
|
Names[RTLIB::MUL_PPCF128] = "__gcc_qmul";
|
|
|
|
|
Names[RTLIB::DIV_F32] = "__divsf3";
|
|
|
|
|
Names[RTLIB::DIV_F64] = "__divdf3";
|
|
|
|
|
Names[RTLIB::DIV_F80] = "__divxf3";
|
|
|
|
|
Names[RTLIB::DIV_F128] = "__divtf3";
|
|
|
|
|
Names[RTLIB::DIV_PPCF128] = "__gcc_qdiv";
|
|
|
|
|
Names[RTLIB::REM_F32] = "fmodf";
|
|
|
|
|
Names[RTLIB::REM_F64] = "fmod";
|
|
|
|
|
Names[RTLIB::REM_F80] = "fmodl";
|
|
|
|
|
Names[RTLIB::REM_F128] = "fmodl";
|
|
|
|
|
Names[RTLIB::REM_PPCF128] = "fmodl";
|
|
|
|
|
Names[RTLIB::FMA_F32] = "fmaf";
|
|
|
|
|
Names[RTLIB::FMA_F64] = "fma";
|
|
|
|
|
Names[RTLIB::FMA_F80] = "fmal";
|
|
|
|
|
Names[RTLIB::FMA_F128] = "fmal";
|
|
|
|
|
Names[RTLIB::FMA_PPCF128] = "fmal";
|
|
|
|
|
Names[RTLIB::POWI_F32] = "__powisf2";
|
|
|
|
|
Names[RTLIB::POWI_F64] = "__powidf2";
|
|
|
|
|
Names[RTLIB::POWI_F80] = "__powixf2";
|
|
|
|
|
Names[RTLIB::POWI_F128] = "__powitf2";
|
|
|
|
|
Names[RTLIB::POWI_PPCF128] = "__powitf2";
|
|
|
|
|
Names[RTLIB::SQRT_F32] = "sqrtf";
|
|
|
|
|
Names[RTLIB::SQRT_F64] = "sqrt";
|
|
|
|
|
Names[RTLIB::SQRT_F80] = "sqrtl";
|
|
|
|
|
Names[RTLIB::SQRT_F128] = "sqrtl";
|
|
|
|
|
Names[RTLIB::SQRT_PPCF128] = "sqrtl";
|
|
|
|
|
Names[RTLIB::LOG_F32] = "logf";
|
|
|
|
|
Names[RTLIB::LOG_F64] = "log";
|
|
|
|
|
Names[RTLIB::LOG_F80] = "logl";
|
|
|
|
|
Names[RTLIB::LOG_F128] = "logl";
|
|
|
|
|
Names[RTLIB::LOG_PPCF128] = "logl";
|
|
|
|
|
Names[RTLIB::LOG2_F32] = "log2f";
|
|
|
|
|
Names[RTLIB::LOG2_F64] = "log2";
|
|
|
|
|
Names[RTLIB::LOG2_F80] = "log2l";
|
|
|
|
|
Names[RTLIB::LOG2_F128] = "log2l";
|
|
|
|
|
Names[RTLIB::LOG2_PPCF128] = "log2l";
|
|
|
|
|
Names[RTLIB::LOG10_F32] = "log10f";
|
|
|
|
|
Names[RTLIB::LOG10_F64] = "log10";
|
|
|
|
|
Names[RTLIB::LOG10_F80] = "log10l";
|
|
|
|
|
Names[RTLIB::LOG10_F128] = "log10l";
|
|
|
|
|
Names[RTLIB::LOG10_PPCF128] = "log10l";
|
|
|
|
|
Names[RTLIB::EXP_F32] = "expf";
|
|
|
|
|
Names[RTLIB::EXP_F64] = "exp";
|
|
|
|
|
Names[RTLIB::EXP_F80] = "expl";
|
|
|
|
|
Names[RTLIB::EXP_F128] = "expl";
|
|
|
|
|
Names[RTLIB::EXP_PPCF128] = "expl";
|
|
|
|
|
Names[RTLIB::EXP2_F32] = "exp2f";
|
|
|
|
|
Names[RTLIB::EXP2_F64] = "exp2";
|
|
|
|
|
Names[RTLIB::EXP2_F80] = "exp2l";
|
|
|
|
|
Names[RTLIB::EXP2_F128] = "exp2l";
|
|
|
|
|
Names[RTLIB::EXP2_PPCF128] = "exp2l";
|
|
|
|
|
Names[RTLIB::SIN_F32] = "sinf";
|
|
|
|
|
Names[RTLIB::SIN_F64] = "sin";
|
|
|
|
|
Names[RTLIB::SIN_F80] = "sinl";
|
|
|
|
|
Names[RTLIB::SIN_F128] = "sinl";
|
|
|
|
|
Names[RTLIB::SIN_PPCF128] = "sinl";
|
|
|
|
|
Names[RTLIB::COS_F32] = "cosf";
|
|
|
|
|
Names[RTLIB::COS_F64] = "cos";
|
|
|
|
|
Names[RTLIB::COS_F80] = "cosl";
|
|
|
|
|
Names[RTLIB::COS_F128] = "cosl";
|
|
|
|
|
Names[RTLIB::COS_PPCF128] = "cosl";
|
|
|
|
|
Names[RTLIB::POW_F32] = "powf";
|
|
|
|
|
Names[RTLIB::POW_F64] = "pow";
|
|
|
|
|
Names[RTLIB::POW_F80] = "powl";
|
|
|
|
|
Names[RTLIB::POW_F128] = "powl";
|
|
|
|
|
Names[RTLIB::POW_PPCF128] = "powl";
|
|
|
|
|
Names[RTLIB::CEIL_F32] = "ceilf";
|
|
|
|
|
Names[RTLIB::CEIL_F64] = "ceil";
|
|
|
|
|
Names[RTLIB::CEIL_F80] = "ceill";
|
|
|
|
|
Names[RTLIB::CEIL_F128] = "ceill";
|
|
|
|
|
Names[RTLIB::CEIL_PPCF128] = "ceill";
|
|
|
|
|
Names[RTLIB::TRUNC_F32] = "truncf";
|
|
|
|
|
Names[RTLIB::TRUNC_F64] = "trunc";
|
|
|
|
|
Names[RTLIB::TRUNC_F80] = "truncl";
|
|
|
|
|
Names[RTLIB::TRUNC_F128] = "truncl";
|
|
|
|
|
Names[RTLIB::TRUNC_PPCF128] = "truncl";
|
|
|
|
|
Names[RTLIB::RINT_F32] = "rintf";
|
|
|
|
|
Names[RTLIB::RINT_F64] = "rint";
|
|
|
|
|
Names[RTLIB::RINT_F80] = "rintl";
|
|
|
|
|
Names[RTLIB::RINT_F128] = "rintl";
|
|
|
|
|
Names[RTLIB::RINT_PPCF128] = "rintl";
|
|
|
|
|
Names[RTLIB::NEARBYINT_F32] = "nearbyintf";
|
|
|
|
|
Names[RTLIB::NEARBYINT_F64] = "nearbyint";
|
|
|
|
|
Names[RTLIB::NEARBYINT_F80] = "nearbyintl";
|
|
|
|
|
Names[RTLIB::NEARBYINT_F128] = "nearbyintl";
|
|
|
|
|
Names[RTLIB::NEARBYINT_PPCF128] = "nearbyintl";
|
2013-08-07 22:49:12 +00:00
|
|
|
Names[RTLIB::ROUND_F32] = "roundf";
|
|
|
|
|
Names[RTLIB::ROUND_F64] = "round";
|
|
|
|
|
Names[RTLIB::ROUND_F80] = "roundl";
|
|
|
|
|
Names[RTLIB::ROUND_F128] = "roundl";
|
|
|
|
|
Names[RTLIB::ROUND_PPCF128] = "roundl";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FLOOR_F32] = "floorf";
|
|
|
|
|
Names[RTLIB::FLOOR_F64] = "floor";
|
|
|
|
|
Names[RTLIB::FLOOR_F80] = "floorl";
|
|
|
|
|
Names[RTLIB::FLOOR_F128] = "floorl";
|
|
|
|
|
Names[RTLIB::FLOOR_PPCF128] = "floorl";
|
2014-10-21 23:01:01 +00:00
|
|
|
Names[RTLIB::FMIN_F32] = "fminf";
|
|
|
|
|
Names[RTLIB::FMIN_F64] = "fmin";
|
|
|
|
|
Names[RTLIB::FMIN_F80] = "fminl";
|
|
|
|
|
Names[RTLIB::FMIN_F128] = "fminl";
|
|
|
|
|
Names[RTLIB::FMIN_PPCF128] = "fminl";
|
|
|
|
|
Names[RTLIB::FMAX_F32] = "fmaxf";
|
|
|
|
|
Names[RTLIB::FMAX_F64] = "fmax";
|
|
|
|
|
Names[RTLIB::FMAX_F80] = "fmaxl";
|
|
|
|
|
Names[RTLIB::FMAX_F128] = "fmaxl";
|
|
|
|
|
Names[RTLIB::FMAX_PPCF128] = "fmaxl";
|
2014-03-29 09:03:18 +00:00
|
|
|
Names[RTLIB::ROUND_F32] = "roundf";
|
|
|
|
|
Names[RTLIB::ROUND_F64] = "round";
|
|
|
|
|
Names[RTLIB::ROUND_F80] = "roundl";
|
|
|
|
|
Names[RTLIB::ROUND_F128] = "roundl";
|
|
|
|
|
Names[RTLIB::ROUND_PPCF128] = "roundl";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::COPYSIGN_F32] = "copysignf";
|
|
|
|
|
Names[RTLIB::COPYSIGN_F64] = "copysign";
|
|
|
|
|
Names[RTLIB::COPYSIGN_F80] = "copysignl";
|
|
|
|
|
Names[RTLIB::COPYSIGN_F128] = "copysignl";
|
|
|
|
|
Names[RTLIB::COPYSIGN_PPCF128] = "copysignl";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::FPEXT_F32_PPCF128] = "__gcc_stoq";
|
|
|
|
|
Names[RTLIB::FPEXT_F64_PPCF128] = "__gcc_dtoq";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FPEXT_F64_F128] = "__extenddftf2";
|
|
|
|
|
Names[RTLIB::FPEXT_F32_F128] = "__extendsftf2";
|
|
|
|
|
Names[RTLIB::FPEXT_F32_F64] = "__extendsfdf2";
|
2016-04-12 22:32:47 +00:00
|
|
|
if (TT.isOSDarwin()) {
|
|
|
|
|
// For f16/f32 conversions, Darwin uses the standard naming scheme, instead
|
|
|
|
|
// of the gnueabi-style __gnu_*_ieee.
|
|
|
|
|
// FIXME: What about other targets?
|
|
|
|
|
Names[RTLIB::FPEXT_F16_F32] = "__extendhfsf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F32_F16] = "__truncsfhf2";
|
|
|
|
|
} else {
|
|
|
|
|
Names[RTLIB::FPEXT_F16_F32] = "__gnu_h2f_ieee";
|
|
|
|
|
Names[RTLIB::FPROUND_F32_F16] = "__gnu_f2h_ieee";
|
|
|
|
|
}
|
2014-07-17 11:12:12 +00:00
|
|
|
Names[RTLIB::FPROUND_F64_F16] = "__truncdfhf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F80_F16] = "__truncxfhf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F128_F16] = "__trunctfhf2";
|
|
|
|
|
Names[RTLIB::FPROUND_PPCF128_F16] = "__trunctfhf2";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FPROUND_F64_F32] = "__truncdfsf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F80_F32] = "__truncxfsf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F128_F32] = "__trunctfsf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::FPROUND_PPCF128_F32] = "__gcc_qtos";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FPROUND_F80_F64] = "__truncxfdf2";
|
|
|
|
|
Names[RTLIB::FPROUND_F128_F64] = "__trunctfdf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::FPROUND_PPCF128_F64] = "__gcc_qtod";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FPTOSINT_F32_I32] = "__fixsfsi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F32_I64] = "__fixsfdi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F32_I128] = "__fixsfti";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F64_I32] = "__fixdfsi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F64_I64] = "__fixdfdi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F64_I128] = "__fixdfti";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F80_I32] = "__fixxfsi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F80_I64] = "__fixxfdi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F80_I128] = "__fixxfti";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F128_I32] = "__fixtfsi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F128_I64] = "__fixtfdi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_F128_I128] = "__fixtfti";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::FPTOSINT_PPCF128_I32] = "__gcc_qtou";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::FPTOSINT_PPCF128_I64] = "__fixtfdi";
|
|
|
|
|
Names[RTLIB::FPTOSINT_PPCF128_I128] = "__fixtfti";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F32_I32] = "__fixunssfsi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F32_I64] = "__fixunssfdi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F32_I128] = "__fixunssfti";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F64_I32] = "__fixunsdfsi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F64_I64] = "__fixunsdfdi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F64_I128] = "__fixunsdfti";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F80_I32] = "__fixunsxfsi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F80_I64] = "__fixunsxfdi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F80_I128] = "__fixunsxfti";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F128_I32] = "__fixunstfsi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F128_I64] = "__fixunstfdi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_F128_I128] = "__fixunstfti";
|
|
|
|
|
Names[RTLIB::FPTOUINT_PPCF128_I32] = "__fixunstfsi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_PPCF128_I64] = "__fixunstfdi";
|
|
|
|
|
Names[RTLIB::FPTOUINT_PPCF128_I128] = "__fixunstfti";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I32_F32] = "__floatsisf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I32_F64] = "__floatsidf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I32_F80] = "__floatsixf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I32_F128] = "__floatsitf";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::SINTTOFP_I32_PPCF128] = "__gcc_itoq";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SINTTOFP_I64_F32] = "__floatdisf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I64_F64] = "__floatdidf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I64_F80] = "__floatdixf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I64_F128] = "__floatditf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I64_PPCF128] = "__floatditf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I128_F32] = "__floattisf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I128_F64] = "__floattidf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I128_F80] = "__floattixf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I128_F128] = "__floattitf";
|
|
|
|
|
Names[RTLIB::SINTTOFP_I128_PPCF128] = "__floattitf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I32_F32] = "__floatunsisf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I32_F64] = "__floatunsidf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I32_F80] = "__floatunsixf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I32_F128] = "__floatunsitf";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::UINTTOFP_I32_PPCF128] = "__gcc_utoq";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::UINTTOFP_I64_F32] = "__floatundisf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I64_F64] = "__floatundidf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I64_F80] = "__floatundixf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I64_F128] = "__floatunditf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I64_PPCF128] = "__floatunditf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I128_F32] = "__floatuntisf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I128_F64] = "__floatuntidf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I128_F80] = "__floatuntixf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I128_F128] = "__floatuntitf";
|
|
|
|
|
Names[RTLIB::UINTTOFP_I128_PPCF128] = "__floatuntitf";
|
|
|
|
|
Names[RTLIB::OEQ_F32] = "__eqsf2";
|
|
|
|
|
Names[RTLIB::OEQ_F64] = "__eqdf2";
|
|
|
|
|
Names[RTLIB::OEQ_F128] = "__eqtf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::OEQ_PPCF128] = "__gcc_qeq";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::UNE_F32] = "__nesf2";
|
|
|
|
|
Names[RTLIB::UNE_F64] = "__nedf2";
|
|
|
|
|
Names[RTLIB::UNE_F128] = "__netf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::UNE_PPCF128] = "__gcc_qne";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::OGE_F32] = "__gesf2";
|
|
|
|
|
Names[RTLIB::OGE_F64] = "__gedf2";
|
|
|
|
|
Names[RTLIB::OGE_F128] = "__getf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::OGE_PPCF128] = "__gcc_qge";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::OLT_F32] = "__ltsf2";
|
|
|
|
|
Names[RTLIB::OLT_F64] = "__ltdf2";
|
|
|
|
|
Names[RTLIB::OLT_F128] = "__lttf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::OLT_PPCF128] = "__gcc_qlt";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::OLE_F32] = "__lesf2";
|
|
|
|
|
Names[RTLIB::OLE_F64] = "__ledf2";
|
|
|
|
|
Names[RTLIB::OLE_F128] = "__letf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::OLE_PPCF128] = "__gcc_qle";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::OGT_F32] = "__gtsf2";
|
|
|
|
|
Names[RTLIB::OGT_F64] = "__gtdf2";
|
|
|
|
|
Names[RTLIB::OGT_F128] = "__gttf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::OGT_PPCF128] = "__gcc_qgt";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::UO_F32] = "__unordsf2";
|
|
|
|
|
Names[RTLIB::UO_F64] = "__unorddf2";
|
|
|
|
|
Names[RTLIB::UO_F128] = "__unordtf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::UO_PPCF128] = "__gcc_qunord";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::O_F32] = "__unordsf2";
|
|
|
|
|
Names[RTLIB::O_F64] = "__unorddf2";
|
|
|
|
|
Names[RTLIB::O_F128] = "__unordtf2";
|
2016-02-04 14:43:50 +00:00
|
|
|
Names[RTLIB::O_PPCF128] = "__gcc_qunord";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::MEMCPY] = "memcpy";
|
|
|
|
|
Names[RTLIB::MEMMOVE] = "memmove";
|
|
|
|
|
Names[RTLIB::MEMSET] = "memset";
|
2017-06-16 14:43:59 +00:00
|
|
|
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_1] =
|
|
|
|
|
"__llvm_memcpy_element_unordered_atomic_1";
|
|
|
|
|
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_2] =
|
|
|
|
|
"__llvm_memcpy_element_unordered_atomic_2";
|
|
|
|
|
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_4] =
|
|
|
|
|
"__llvm_memcpy_element_unordered_atomic_4";
|
|
|
|
|
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_8] =
|
|
|
|
|
"__llvm_memcpy_element_unordered_atomic_8";
|
|
|
|
|
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_16] =
|
|
|
|
|
"__llvm_memcpy_element_unordered_atomic_16";
|
2017-07-12 15:25:26 +00:00
|
|
|
Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1] =
|
|
|
|
|
"__llvm_memmove_element_unordered_atomic_1";
|
|
|
|
|
Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2] =
|
|
|
|
|
"__llvm_memmove_element_unordered_atomic_2";
|
|
|
|
|
Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4] =
|
|
|
|
|
"__llvm_memmove_element_unordered_atomic_4";
|
|
|
|
|
Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8] =
|
|
|
|
|
"__llvm_memmove_element_unordered_atomic_8";
|
|
|
|
|
Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16] =
|
|
|
|
|
"__llvm_memmove_element_unordered_atomic_16";
|
Add element atomic memset intrinsic
Summary: Continuing the work from https://reviews.llvm.org/D33240, this change introduces an element unordered-atomic memset intrinsic. This intrinsic is essentially memset with the implementation requirement that all stores used for the assignment are done with unordered-atomic stores of a given element size.
Reviewers: eli.friedman, reames, mkazantsev, skatkov
Reviewed By: reames
Subscribers: jfb, dschuff, sbc100, jgravelle-google, aheejin, efriedma, llvm-commits
Differential Revision: https://reviews.llvm.org/D34885
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@307854 91177308-0d34-0410-b5e6-96231b3b80d8
2017-07-12 21:57:23 +00:00
|
|
|
Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_1] =
|
|
|
|
|
"__llvm_memset_element_unordered_atomic_1";
|
|
|
|
|
Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_2] =
|
|
|
|
|
"__llvm_memset_element_unordered_atomic_2";
|
|
|
|
|
Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_4] =
|
|
|
|
|
"__llvm_memset_element_unordered_atomic_4";
|
|
|
|
|
Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_8] =
|
|
|
|
|
"__llvm_memset_element_unordered_atomic_8";
|
|
|
|
|
Names[RTLIB::MEMSET_ELEMENT_UNORDERED_ATOMIC_16] =
|
|
|
|
|
"__llvm_memset_element_unordered_atomic_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::UNWIND_RESUME] = "_Unwind_Resume";
|
|
|
|
|
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1] = "__sync_val_compare_and_swap_1";
|
|
|
|
|
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2] = "__sync_val_compare_and_swap_2";
|
|
|
|
|
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4] = "__sync_val_compare_and_swap_4";
|
|
|
|
|
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8] = "__sync_val_compare_and_swap_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16] = "__sync_val_compare_and_swap_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_LOCK_TEST_AND_SET_1] = "__sync_lock_test_and_set_1";
|
|
|
|
|
Names[RTLIB::SYNC_LOCK_TEST_AND_SET_2] = "__sync_lock_test_and_set_2";
|
|
|
|
|
Names[RTLIB::SYNC_LOCK_TEST_AND_SET_4] = "__sync_lock_test_and_set_4";
|
|
|
|
|
Names[RTLIB::SYNC_LOCK_TEST_AND_SET_8] = "__sync_lock_test_and_set_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_LOCK_TEST_AND_SET_16] = "__sync_lock_test_and_set_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_ADD_1] = "__sync_fetch_and_add_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_ADD_2] = "__sync_fetch_and_add_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_ADD_4] = "__sync_fetch_and_add_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_ADD_8] = "__sync_fetch_and_add_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_ADD_16] = "__sync_fetch_and_add_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_SUB_1] = "__sync_fetch_and_sub_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_SUB_2] = "__sync_fetch_and_sub_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_SUB_4] = "__sync_fetch_and_sub_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_SUB_8] = "__sync_fetch_and_sub_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_SUB_16] = "__sync_fetch_and_sub_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_AND_1] = "__sync_fetch_and_and_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_AND_2] = "__sync_fetch_and_and_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_AND_4] = "__sync_fetch_and_and_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_AND_8] = "__sync_fetch_and_and_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_AND_16] = "__sync_fetch_and_and_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_OR_1] = "__sync_fetch_and_or_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_OR_2] = "__sync_fetch_and_or_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_OR_4] = "__sync_fetch_and_or_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_OR_8] = "__sync_fetch_and_or_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_OR_16] = "__sync_fetch_and_or_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_XOR_1] = "__sync_fetch_and_xor_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_XOR_2] = "__sync_fetch_and_xor_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_XOR_4] = "__sync_fetch_and_xor_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_XOR_8] = "__sync_fetch_and_xor_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_XOR_16] = "__sync_fetch_and_xor_16";
|
2013-01-11 20:05:37 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_NAND_1] = "__sync_fetch_and_nand_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_NAND_2] = "__sync_fetch_and_nand_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_NAND_4] = "__sync_fetch_and_nand_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_NAND_8] = "__sync_fetch_and_nand_8";
|
2013-10-18 08:03:43 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_NAND_16] = "__sync_fetch_and_nand_16";
|
2013-10-25 09:30:20 +00:00
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MAX_1] = "__sync_fetch_and_max_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MAX_2] = "__sync_fetch_and_max_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MAX_4] = "__sync_fetch_and_max_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MAX_8] = "__sync_fetch_and_max_8";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MAX_16] = "__sync_fetch_and_max_16";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMAX_1] = "__sync_fetch_and_umax_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMAX_2] = "__sync_fetch_and_umax_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMAX_4] = "__sync_fetch_and_umax_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMAX_8] = "__sync_fetch_and_umax_8";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMAX_16] = "__sync_fetch_and_umax_16";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MIN_1] = "__sync_fetch_and_min_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MIN_2] = "__sync_fetch_and_min_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MIN_4] = "__sync_fetch_and_min_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MIN_8] = "__sync_fetch_and_min_8";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_MIN_16] = "__sync_fetch_and_min_16";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMIN_1] = "__sync_fetch_and_umin_1";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMIN_2] = "__sync_fetch_and_umin_2";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMIN_4] = "__sync_fetch_and_umin_4";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMIN_8] = "__sync_fetch_and_umin_8";
|
|
|
|
|
Names[RTLIB::SYNC_FETCH_AND_UMIN_16] = "__sync_fetch_and_umin_16";
|
Add __atomic_* lowering to AtomicExpandPass.
(Recommit of r266002, with r266011, r266016, and not accidentally
including an extra unused/uninitialized element in LibcallRoutineNames)
AtomicExpandPass can now lower atomic load, atomic store, atomicrmw, and
cmpxchg instructions to __atomic_* library calls, when the target
doesn't support atomics of a given size.
This is the first step towards moving all atomic lowering from clang
into llvm. When all is done, the behavior of __sync_* builtins,
__atomic_* builtins, and C11 atomics will be unified.
Previously LLVM would pass everything through to the ISelLowering
code. There, unsupported atomic instructions would turn into __sync_*
library calls. Because of that behavior, Clang currently avoids emitting
llvm IR atomic instructions when this would happen, and emits __atomic_*
library functions itself, in the frontend.
This change makes LLVM able to emit __atomic_* libcalls, and thus will
eventually allow clang to depend on LLVM to do the right thing.
It is advantageous to do the new lowering to atomic libcalls in
AtomicExpandPass, before ISel time, because it's important that all
atomic operations for a given size either lower to __atomic_*
libcalls (which may use locks), or native instructions which won't. No
mixing and matching.
At the moment, this code is enabled only for SPARC, as a
demonstration. The next commit will expand support to all of the other
targets.
Differential Revision: http://reviews.llvm.org/D18200
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@266115 91177308-0d34-0410-b5e6-96231b3b80d8
2016-04-12 20:18:48 +00:00
|
|
|
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD] = "__atomic_load";
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD_1] = "__atomic_load_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD_2] = "__atomic_load_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD_4] = "__atomic_load_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD_8] = "__atomic_load_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_LOAD_16] = "__atomic_load_16";
|
|
|
|
|
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE] = "__atomic_store";
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE_1] = "__atomic_store_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE_2] = "__atomic_store_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE_4] = "__atomic_store_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE_8] = "__atomic_store_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_STORE_16] = "__atomic_store_16";
|
|
|
|
|
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE] = "__atomic_exchange";
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE_1] = "__atomic_exchange_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE_2] = "__atomic_exchange_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE_4] = "__atomic_exchange_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE_8] = "__atomic_exchange_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_EXCHANGE_16] = "__atomic_exchange_16";
|
|
|
|
|
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE] = "__atomic_compare_exchange";
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE_1] = "__atomic_compare_exchange_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE_2] = "__atomic_compare_exchange_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE_4] = "__atomic_compare_exchange_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE_8] = "__atomic_compare_exchange_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_COMPARE_EXCHANGE_16] = "__atomic_compare_exchange_16";
|
|
|
|
|
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_ADD_1] = "__atomic_fetch_add_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_ADD_2] = "__atomic_fetch_add_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_ADD_4] = "__atomic_fetch_add_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_ADD_8] = "__atomic_fetch_add_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_ADD_16] = "__atomic_fetch_add_16";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_SUB_1] = "__atomic_fetch_sub_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_SUB_2] = "__atomic_fetch_sub_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_SUB_4] = "__atomic_fetch_sub_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_SUB_8] = "__atomic_fetch_sub_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_SUB_16] = "__atomic_fetch_sub_16";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_AND_1] = "__atomic_fetch_and_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_AND_2] = "__atomic_fetch_and_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_AND_4] = "__atomic_fetch_and_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_AND_8] = "__atomic_fetch_and_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_AND_16] = "__atomic_fetch_and_16";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_OR_1] = "__atomic_fetch_or_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_OR_2] = "__atomic_fetch_or_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_OR_4] = "__atomic_fetch_or_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_OR_8] = "__atomic_fetch_or_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_OR_16] = "__atomic_fetch_or_16";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_XOR_1] = "__atomic_fetch_xor_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_XOR_2] = "__atomic_fetch_xor_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_XOR_4] = "__atomic_fetch_xor_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_XOR_8] = "__atomic_fetch_xor_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_XOR_16] = "__atomic_fetch_xor_16";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_NAND_1] = "__atomic_fetch_nand_1";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_NAND_2] = "__atomic_fetch_nand_2";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_NAND_4] = "__atomic_fetch_nand_4";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_NAND_8] = "__atomic_fetch_nand_8";
|
|
|
|
|
Names[RTLIB::ATOMIC_FETCH_NAND_16] = "__atomic_fetch_nand_16";
|
|
|
|
|
|
2016-06-21 12:29:03 +00:00
|
|
|
if (TT.isGNUEnvironment()) {
|
2013-02-15 18:45:18 +00:00
|
|
|
Names[RTLIB::SINCOS_F32] = "sincosf";
|
|
|
|
|
Names[RTLIB::SINCOS_F64] = "sincos";
|
|
|
|
|
Names[RTLIB::SINCOS_F80] = "sincosl";
|
|
|
|
|
Names[RTLIB::SINCOS_F128] = "sincosl";
|
|
|
|
|
Names[RTLIB::SINCOS_PPCF128] = "sincosl";
|
|
|
|
|
}
|
2013-08-12 18:45:38 +00:00
|
|
|
|
2014-11-29 19:18:21 +00:00
|
|
|
if (!TT.isOSOpenBSD()) {
|
2013-08-12 18:45:38 +00:00
|
|
|
Names[RTLIB::STACKPROTECTOR_CHECK_FAIL] = "__stack_chk_fail";
|
2015-05-14 01:00:51 +00:00
|
|
|
}
|
2016-03-24 20:23:29 +00:00
|
|
|
|
|
|
|
|
Names[RTLIB::DEOPTIMIZE] = "__llvm_deoptimize";
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2016-09-07 17:56:09 +00:00
|
|
|
/// Set default libcall CallingConvs.
|
2016-09-09 20:11:31 +00:00
|
|
|
static void InitLibcallCallingConvs(CallingConv::ID *CCs) {
|
2016-09-07 17:56:09 +00:00
|
|
|
for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
|
|
|
|
|
CCs[LC] = CallingConv::C;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getFPEXT - Return the FPEXT_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) {
|
2014-07-21 09:13:56 +00:00
|
|
|
if (OpVT == MVT::f16) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return FPEXT_F16_F32;
|
|
|
|
|
} else if (OpVT == MVT::f32) {
|
2013-01-11 20:05:37 +00:00
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return FPEXT_F32_F64;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return FPEXT_F32_F128;
|
2016-02-04 14:43:50 +00:00
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return FPEXT_F32_PPCF128;
|
2013-01-11 20:05:37 +00:00
|
|
|
} else if (OpVT == MVT::f64) {
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return FPEXT_F64_F128;
|
2016-02-04 14:43:50 +00:00
|
|
|
else if (RetVT == MVT::ppcf128)
|
|
|
|
|
return FPEXT_F64_PPCF128;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getFPROUND - Return the FPROUND_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) {
|
2014-07-17 11:12:12 +00:00
|
|
|
if (RetVT == MVT::f16) {
|
|
|
|
|
if (OpVT == MVT::f32)
|
|
|
|
|
return FPROUND_F32_F16;
|
|
|
|
|
if (OpVT == MVT::f64)
|
|
|
|
|
return FPROUND_F64_F16;
|
|
|
|
|
if (OpVT == MVT::f80)
|
|
|
|
|
return FPROUND_F80_F16;
|
|
|
|
|
if (OpVT == MVT::f128)
|
|
|
|
|
return FPROUND_F128_F16;
|
|
|
|
|
if (OpVT == MVT::ppcf128)
|
|
|
|
|
return FPROUND_PPCF128_F16;
|
|
|
|
|
} else if (RetVT == MVT::f32) {
|
2013-01-11 20:05:37 +00:00
|
|
|
if (OpVT == MVT::f64)
|
|
|
|
|
return FPROUND_F64_F32;
|
|
|
|
|
if (OpVT == MVT::f80)
|
|
|
|
|
return FPROUND_F80_F32;
|
|
|
|
|
if (OpVT == MVT::f128)
|
|
|
|
|
return FPROUND_F128_F32;
|
|
|
|
|
if (OpVT == MVT::ppcf128)
|
|
|
|
|
return FPROUND_PPCF128_F32;
|
|
|
|
|
} else if (RetVT == MVT::f64) {
|
|
|
|
|
if (OpVT == MVT::f80)
|
|
|
|
|
return FPROUND_F80_F64;
|
|
|
|
|
if (OpVT == MVT::f128)
|
|
|
|
|
return FPROUND_F128_F64;
|
|
|
|
|
if (OpVT == MVT::ppcf128)
|
|
|
|
|
return FPROUND_PPCF128_F64;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
|
|
|
|
|
if (OpVT == MVT::f32) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOSINT_F32_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOSINT_F32_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOSINT_F32_I128;
|
|
|
|
|
} else if (OpVT == MVT::f64) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOSINT_F64_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOSINT_F64_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOSINT_F64_I128;
|
|
|
|
|
} else if (OpVT == MVT::f80) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOSINT_F80_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOSINT_F80_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOSINT_F80_I128;
|
|
|
|
|
} else if (OpVT == MVT::f128) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOSINT_F128_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOSINT_F128_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOSINT_F128_I128;
|
|
|
|
|
} else if (OpVT == MVT::ppcf128) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOSINT_PPCF128_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOSINT_PPCF128_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOSINT_PPCF128_I128;
|
|
|
|
|
}
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
|
|
|
|
|
if (OpVT == MVT::f32) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOUINT_F32_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOUINT_F32_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOUINT_F32_I128;
|
|
|
|
|
} else if (OpVT == MVT::f64) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOUINT_F64_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOUINT_F64_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOUINT_F64_I128;
|
|
|
|
|
} else if (OpVT == MVT::f80) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOUINT_F80_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOUINT_F80_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOUINT_F80_I128;
|
|
|
|
|
} else if (OpVT == MVT::f128) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOUINT_F128_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOUINT_F128_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOUINT_F128_I128;
|
|
|
|
|
} else if (OpVT == MVT::ppcf128) {
|
|
|
|
|
if (RetVT == MVT::i32)
|
|
|
|
|
return FPTOUINT_PPCF128_I32;
|
|
|
|
|
if (RetVT == MVT::i64)
|
|
|
|
|
return FPTOUINT_PPCF128_I64;
|
|
|
|
|
if (RetVT == MVT::i128)
|
|
|
|
|
return FPTOUINT_PPCF128_I128;
|
|
|
|
|
}
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
|
|
|
|
|
if (OpVT == MVT::i32) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return SINTTOFP_I32_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return SINTTOFP_I32_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return SINTTOFP_I32_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return SINTTOFP_I32_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return SINTTOFP_I32_PPCF128;
|
|
|
|
|
} else if (OpVT == MVT::i64) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return SINTTOFP_I64_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return SINTTOFP_I64_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return SINTTOFP_I64_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return SINTTOFP_I64_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return SINTTOFP_I64_PPCF128;
|
|
|
|
|
} else if (OpVT == MVT::i128) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return SINTTOFP_I128_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return SINTTOFP_I128_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return SINTTOFP_I128_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return SINTTOFP_I128_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return SINTTOFP_I128_PPCF128;
|
|
|
|
|
}
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
|
|
|
|
|
/// UNKNOWN_LIBCALL if there is none.
|
|
|
|
|
RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
|
|
|
|
|
if (OpVT == MVT::i32) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return UINTTOFP_I32_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return UINTTOFP_I32_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return UINTTOFP_I32_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return UINTTOFP_I32_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return UINTTOFP_I32_PPCF128;
|
|
|
|
|
} else if (OpVT == MVT::i64) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return UINTTOFP_I64_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return UINTTOFP_I64_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return UINTTOFP_I64_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return UINTTOFP_I64_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return UINTTOFP_I64_PPCF128;
|
|
|
|
|
} else if (OpVT == MVT::i128) {
|
|
|
|
|
if (RetVT == MVT::f32)
|
|
|
|
|
return UINTTOFP_I128_F32;
|
|
|
|
|
if (RetVT == MVT::f64)
|
|
|
|
|
return UINTTOFP_I128_F64;
|
|
|
|
|
if (RetVT == MVT::f80)
|
|
|
|
|
return UINTTOFP_I128_F80;
|
|
|
|
|
if (RetVT == MVT::f128)
|
|
|
|
|
return UINTTOFP_I128_F128;
|
|
|
|
|
if (RetVT == MVT::ppcf128)
|
|
|
|
|
return UINTTOFP_I128_PPCF128;
|
|
|
|
|
}
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-16 22:12:04 +00:00
|
|
|
RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
|
2015-03-05 20:04:29 +00:00
|
|
|
#define OP_TO_LIBCALL(Name, Enum) \
|
|
|
|
|
case Name: \
|
|
|
|
|
switch (VT.SimpleTy) { \
|
|
|
|
|
default: \
|
|
|
|
|
return UNKNOWN_LIBCALL; \
|
|
|
|
|
case MVT::i8: \
|
|
|
|
|
return Enum##_1; \
|
|
|
|
|
case MVT::i16: \
|
|
|
|
|
return Enum##_2; \
|
|
|
|
|
case MVT::i32: \
|
|
|
|
|
return Enum##_4; \
|
|
|
|
|
case MVT::i64: \
|
|
|
|
|
return Enum##_8; \
|
|
|
|
|
case MVT::i128: \
|
|
|
|
|
return Enum##_16; \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
switch (Opc) {
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
|
|
|
|
|
OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#undef OP_TO_LIBCALL
|
|
|
|
|
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
|
2017-06-16 14:43:59 +00:00
|
|
|
RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
|
2016-12-29 14:31:07 +00:00
|
|
|
switch (ElementSize) {
|
|
|
|
|
case 1:
|
2017-06-16 14:43:59 +00:00
|
|
|
return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
|
2016-12-29 14:31:07 +00:00
|
|
|
case 2:
|
2017-06-16 14:43:59 +00:00
|
|
|
return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
|
2016-12-29 14:31:07 +00:00
|
|
|
case 4:
|
2017-06-16 14:43:59 +00:00
|
|
|
return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
|
2016-12-29 14:31:07 +00:00
|
|
|
case 8:
|
2017-06-16 14:43:59 +00:00
|
|
|
return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
|
2016-12-29 14:31:07 +00:00
|
|
|
case 16:
|
2017-06-16 14:43:59 +00:00
|
|
|
return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
|
2016-12-29 14:31:07 +00:00
|
|
|
default:
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-07-12 15:25:26 +00:00
|
|
|
RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
|
|
|
|
|
switch (ElementSize) {
|
|
|
|
|
case 1:
|
|
|
|
|
return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
|
|
|
|
|
case 2:
|
|
|
|
|
return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
|
|
|
|
|
case 4:
|
|
|
|
|
return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
|
|
|
|
|
case 8:
|
|
|
|
|
return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
|
|
|
|
|
case 16:
|
|
|
|
|
return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
|
|
|
|
|
default:
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
Add element atomic memset intrinsic
Summary: Continuing the work from https://reviews.llvm.org/D33240, this change introduces an element unordered-atomic memset intrinsic. This intrinsic is essentially memset with the implementation requirement that all stores used for the assignment are done with unordered-atomic stores of a given element size.
Reviewers: eli.friedman, reames, mkazantsev, skatkov
Reviewed By: reames
Subscribers: jfb, dschuff, sbc100, jgravelle-google, aheejin, efriedma, llvm-commits
Differential Revision: https://reviews.llvm.org/D34885
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@307854 91177308-0d34-0410-b5e6-96231b3b80d8
2017-07-12 21:57:23 +00:00
|
|
|
RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
|
|
|
|
|
switch (ElementSize) {
|
|
|
|
|
case 1:
|
|
|
|
|
return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
|
|
|
|
|
case 2:
|
|
|
|
|
return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
|
|
|
|
|
case 4:
|
|
|
|
|
return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
|
|
|
|
|
case 8:
|
|
|
|
|
return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
|
|
|
|
|
case 16:
|
|
|
|
|
return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
|
|
|
|
|
default:
|
|
|
|
|
return UNKNOWN_LIBCALL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
/// InitCmpLibcallCCs - Set default comparison libcall CC.
|
|
|
|
|
///
|
|
|
|
|
static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
|
|
|
|
|
memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL);
|
|
|
|
|
CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
|
|
|
|
|
CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
|
|
|
|
|
CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::UNE_F32] = ISD::SETNE;
|
|
|
|
|
CCs[RTLIB::UNE_F64] = ISD::SETNE;
|
|
|
|
|
CCs[RTLIB::UNE_F128] = ISD::SETNE;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::OGE_F32] = ISD::SETGE;
|
|
|
|
|
CCs[RTLIB::OGE_F64] = ISD::SETGE;
|
|
|
|
|
CCs[RTLIB::OGE_F128] = ISD::SETGE;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::OLT_F32] = ISD::SETLT;
|
|
|
|
|
CCs[RTLIB::OLT_F64] = ISD::SETLT;
|
|
|
|
|
CCs[RTLIB::OLT_F128] = ISD::SETLT;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::OLE_F32] = ISD::SETLE;
|
|
|
|
|
CCs[RTLIB::OLE_F64] = ISD::SETLE;
|
|
|
|
|
CCs[RTLIB::OLE_F128] = ISD::SETLE;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::OGT_F32] = ISD::SETGT;
|
|
|
|
|
CCs[RTLIB::OGT_F64] = ISD::SETGT;
|
|
|
|
|
CCs[RTLIB::OGT_F128] = ISD::SETGT;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::UO_F32] = ISD::SETNE;
|
|
|
|
|
CCs[RTLIB::UO_F64] = ISD::SETNE;
|
|
|
|
|
CCs[RTLIB::UO_F128] = ISD::SETNE;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
|
2013-01-11 20:05:37 +00:00
|
|
|
CCs[RTLIB::O_F32] = ISD::SETEQ;
|
|
|
|
|
CCs[RTLIB::O_F64] = ISD::SETEQ;
|
|
|
|
|
CCs[RTLIB::O_F128] = ISD::SETEQ;
|
2016-02-04 14:43:50 +00:00
|
|
|
CCs[RTLIB::O_PPCF128] = ISD::SETEQ;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2014-11-13 21:29:21 +00:00
|
|
|
/// NOTE: The TargetMachine owns TLOF.
|
2015-03-10 02:37:25 +00:00
|
|
|
TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
|
2013-04-05 21:52:40 +00:00
|
|
|
initActions();
|
|
|
|
|
|
|
|
|
|
// Perform these initializations only once.
|
2017-05-31 17:12:38 +00:00
|
|
|
MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove =
|
|
|
|
|
MaxLoadsPerMemcmp = 8;
|
|
|
|
|
MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize =
|
|
|
|
|
MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4;
|
2013-04-05 21:52:40 +00:00
|
|
|
UseUnderscoreSetJmp = false;
|
|
|
|
|
UseUnderscoreLongJmp = false;
|
2014-01-02 21:13:43 +00:00
|
|
|
HasMultipleConditionRegisters = false;
|
2014-04-21 22:22:44 +00:00
|
|
|
HasExtractBitsInsn = false;
|
2015-07-01 18:10:20 +00:00
|
|
|
JumpIsExpensive = JumpIsExpensiveOverride;
|
2013-04-05 21:52:40 +00:00
|
|
|
PredictableSelectIsExpensive = false;
|
[CodeGenPrepare] Reapply r224351 with a fix for the assertion failure:
The type promotion helper does not support vector type, so when make
such it does not kick in in such cases.
Original commit message:
[CodeGenPrepare] Move sign/zero extensions near loads using type promotion.
This patch extends the optimization in CodeGenPrepare that moves a sign/zero
extension near a load when the target can combine them. The optimization may
promote any operations between the extension and the load to make that possible.
Although this optimization may be beneficial for all targets, in particular
AArch64, this is enabled for X86 only as I have not benchmarked it for other
targets yet.
** Context **
Most targets feature extended loads, i.e., loads that perform a zero or sign
extension for free. In that context it is interesting to expose such pattern in
CodeGenPrepare so that the instruction selection pass can form such loads.
Sometimes, this pattern is blocked because of instructions between the load and
the extension. When those instructions are promotable to the extended type, we
can expose this pattern.
** Motivating Example **
Let us consider an example:
define void @foo(i8* %addr1, i32* %addr2, i8 %a, i32 %b) {
%ld = load i8* %addr1
%zextld = zext i8 %ld to i32
%ld2 = load i32* %addr2
%add = add nsw i32 %ld2, %zextld
%sextadd = sext i32 %add to i64
%zexta = zext i8 %a to i32
%addza = add nsw i32 %zexta, %zextld
%sextaddza = sext i32 %addza to i64
%addb = add nsw i32 %b, %zextld
%sextaddb = sext i32 %addb to i64
call void @dummy(i64 %sextadd, i64 %sextaddza, i64 %sextaddb)
ret void
}
As it is, this IR generates the following assembly on x86_64:
[...]
movzbl (%rdi), %eax # zero-extended load
movl (%rsi), %es # plain load
addl %eax, %esi # 32-bit add
movslq %esi, %rdi # sign extend the result of add
movzbl %dl, %edx # zero extend the first argument
addl %eax, %edx # 32-bit add
movslq %edx, %rsi # sign extend the result of add
addl %eax, %ecx # 32-bit add
movslq %ecx, %rdx # sign extend the result of add
[...]
The throughput of this sequence is 7.45 cycles on Ivy Bridge according to IACA.
Now, by promoting the additions to form more extended loads we would generate:
[...]
movzbl (%rdi), %eax # zero-extended load
movslq (%rsi), %rdi # sign-extended load
addq %rax, %rdi # 64-bit add
movzbl %dl, %esi # zero extend the first argument
addq %rax, %rsi # 64-bit add
movslq %ecx, %rdx # sign extend the second argument
addq %rax, %rdx # 64-bit add
[...]
The throughput of this sequence is 6.15 cycles on Ivy Bridge according to IACA.
This kind of sequences happen a lot on code using 32-bit indexes on 64-bit
architectures.
Note: The throughput numbers are similar on Sandy Bridge and Haswell.
** Proposed Solution **
To avoid the penalty of all these sign/zero extensions, we merge them in the
loads at the beginning of the chain of computation by promoting all the chain of
computation on the extended type. The promotion is done if and only if we do not
introduce new extensions, i.e., if we do not degrade the code quality.
To achieve this, we extend the existing “move ext to load” optimization with the
promotion mechanism introduced to match larger patterns for addressing mode
(r200947).
The idea of this extension is to perform the following transformation:
ext(promotableInst1(...(promotableInstN(load))))
=>
promotedInst1(...(promotedInstN(ext(load))))
The promotion mechanism in that optimization is enabled by a new TargetLowering
switch, which is off by default. In other words, by default, the optimization
performs the “move ext to load” optimization as it was before this patch.
** Performance **
Configuration: x86_64: Ivy Bridge fixed at 2900MHz running OS X 10.10.
Tested Optimization Levels: O3/Os
Tests: llvm-testsuite + externals.
Results:
- No regression beside noise.
- Improvements:
CINT2006/473.astar: ~2%
Benchmarks/PAQ8p: ~2%
Misc/perlin: ~3%
The results are consistent for both O3 and Os.
<rdar://problem/18310086>
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@224402 91177308-0d34-0410-b5e6-96231b3b80d8
2014-12-17 01:36:17 +00:00
|
|
|
EnableExtLdPromotion = false;
|
2014-08-08 16:46:53 +00:00
|
|
|
HasFloatingPointExceptions = true;
|
2013-04-05 21:52:40 +00:00
|
|
|
StackPointerRegisterToSaveRestore = 0;
|
|
|
|
|
BooleanContents = UndefinedBooleanContent;
|
2014-07-10 10:18:12 +00:00
|
|
|
BooleanFloatContents = UndefinedBooleanContent;
|
2013-04-05 21:52:40 +00:00
|
|
|
BooleanVectorContents = UndefinedBooleanContent;
|
|
|
|
|
SchedPreferenceInfo = Sched::ILP;
|
|
|
|
|
JumpBufSize = 0;
|
|
|
|
|
JumpBufAlignment = 0;
|
|
|
|
|
MinFunctionAlignment = 0;
|
|
|
|
|
PrefFunctionAlignment = 0;
|
|
|
|
|
PrefLoopAlignment = 0;
|
In visitSTORE, always use FindBetterChain, rather than only when UseAA is enabled.
Recommiting with compiler time improvements
Recommitting after fixup of 32-bit aliasing sign offset bug in DAGCombiner.
* Simplify Consecutive Merge Store Candidate Search
Now that address aliasing is much less conservative, push through
simplified store merging search and chain alias analysis which only
checks for parallel stores through the chain subgraph. This is cleaner
as the separation of non-interfering loads/stores from the
store-merging logic.
When merging stores search up the chain through a single load, and
finds all possible stores by looking down from through a load and a
TokenFactor to all stores visited.
This improves the quality of the output SelectionDAG and the output
Codegen (save perhaps for some ARM cases where we correctly constructs
wider loads, but then promotes them to float operations which appear
but requires more expensive constant generation).
Some minor peephole optimizations to deal with improved SubDAG shapes (listed below)
Additional Minor Changes:
1. Finishes removing unused AliasLoad code
2. Unifies the chain aggregation in the merged stores across code
paths
3. Re-add the Store node to the worklist after calling
SimplifyDemandedBits.
4. Increase GatherAllAliasesMaxDepth from 6 to 18. That number is
arbitrary, but seems sufficient to not cause regressions in
tests.
5. Remove Chain dependencies of Memory operations on CopyfromReg
nodes as these are captured by data dependence
6. Forward loads-store values through tokenfactors containing
{CopyToReg,CopyFromReg} Values.
7. Peephole to convert buildvector of extract_vector_elt to
extract_subvector if possible (see
CodeGen/AArch64/store-merge.ll)
8. Store merging for the ARM target is restricted to 32-bit as
some in some contexts invalid 64-bit operations are being
generated. This can be removed once appropriate checks are
added.
This finishes the change Matt Arsenault started in r246307 and
jyknight's original patch.
Many tests required some changes as memory operations are now
reorderable, improving load-store forwarding. One test in
particular is worth noting:
CodeGen/PowerPC/ppc64-align-long-double.ll - Improved load-store
forwarding converts a load-store pair into a parallel store and
a memory-realized bitcast of the same value. However, because we
lose the sharing of the explicit and implicit store values we
must create another local store. A similar transformation
happens before SelectionDAG as well.
Reviewers: arsenm, hfinkel, tstellarAMD, jyknight, nhaehnle
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@297695 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-14 00:34:14 +00:00
|
|
|
GatherAllAliasesMaxDepth = 18;
|
2013-04-05 21:52:40 +00:00
|
|
|
MinStackArgumentAlignment = 1;
|
Add __atomic_* lowering to AtomicExpandPass.
(Recommit of r266002, with r266011, r266016, and not accidentally
including an extra unused/uninitialized element in LibcallRoutineNames)
AtomicExpandPass can now lower atomic load, atomic store, atomicrmw, and
cmpxchg instructions to __atomic_* library calls, when the target
doesn't support atomics of a given size.
This is the first step towards moving all atomic lowering from clang
into llvm. When all is done, the behavior of __sync_* builtins,
__atomic_* builtins, and C11 atomics will be unified.
Previously LLVM would pass everything through to the ISelLowering
code. There, unsupported atomic instructions would turn into __sync_*
library calls. Because of that behavior, Clang currently avoids emitting
llvm IR atomic instructions when this would happen, and emits __atomic_*
library functions itself, in the frontend.
This change makes LLVM able to emit __atomic_* libcalls, and thus will
eventually allow clang to depend on LLVM to do the right thing.
It is advantageous to do the new lowering to atomic libcalls in
AtomicExpandPass, before ISel time, because it's important that all
atomic operations for a given size either lower to __atomic_*
libcalls (which may use locks), or native instructions which won't. No
mixing and matching.
At the moment, this code is enabled only for SPARC, as a
demonstration. The next commit will expand support to all of the other
targets.
Differential Revision: http://reviews.llvm.org/D18200
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@266115 91177308-0d34-0410-b5e6-96231b3b80d8
2016-04-12 20:18:48 +00:00
|
|
|
// TODO: the default will be switched to 0 in the next commit, along
|
|
|
|
|
// with the Target-specific changes necessary.
|
|
|
|
|
MaxAtomicSizeInBitsSupported = 1024;
|
2013-04-05 21:52:40 +00:00
|
|
|
|
2016-06-17 18:11:48 +00:00
|
|
|
MinCmpXchgSizeInBits = 0;
|
|
|
|
|
|
2016-04-12 22:32:47 +00:00
|
|
|
std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
|
|
|
|
|
|
2015-06-24 13:25:57 +00:00
|
|
|
InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());
|
2013-04-05 21:52:40 +00:00
|
|
|
InitCmpLibcallCCs(CmpLibcallCCs);
|
2016-09-09 20:11:31 +00:00
|
|
|
InitLibcallCallingConvs(LibcallCallingConvs);
|
2013-04-05 21:52:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void TargetLoweringBase::initActions() {
|
2013-01-11 20:05:37 +00:00
|
|
|
// All operations default to being supported.
|
|
|
|
|
memset(OpActions, 0, sizeof(OpActions));
|
|
|
|
|
memset(LoadExtActions, 0, sizeof(LoadExtActions));
|
|
|
|
|
memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
|
|
|
|
|
memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
|
|
|
|
|
memset(CondCodeActions, 0, sizeof(CondCodeActions));
|
2016-04-08 07:10:46 +00:00
|
|
|
std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
|
|
|
|
|
std::fill(std::begin(TargetDAGCombineArray),
|
|
|
|
|
std::end(TargetDAGCombineArray), 0);
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
// Set default actions for various operations.
|
2015-01-07 21:27:10 +00:00
|
|
|
for (MVT VT : MVT::all_valuetypes()) {
|
2013-01-11 20:05:37 +00:00
|
|
|
// Default all indexed load / store to expand.
|
|
|
|
|
for (unsigned IM = (unsigned)ISD::PRE_INC;
|
|
|
|
|
IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
|
2015-01-07 21:27:10 +00:00
|
|
|
setIndexedLoadAction(IM, VT, Expand);
|
|
|
|
|
setIndexedStoreAction(IM, VT, Expand);
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
IR: add "cmpxchg weak" variant to support permitted failure.
This commit adds a weak variant of the cmpxchg operation, as described
in C++11. A cmpxchg instruction with this modifier is permitted to
fail to store, even if the comparison indicated it should.
As a result, cmpxchg instructions must return a flag indicating
success in addition to their original iN value loaded. Thus, for
uniformity *all* cmpxchg instructions now return "{ iN, i1 }". The
second flag is 1 when the store succeeded.
At the DAG level, a new ATOMIC_CMP_SWAP_WITH_SUCCESS node has been
added as the natural representation for the new cmpxchg instructions.
It is a strong cmpxchg.
By default this gets Expanded to the existing ATOMIC_CMP_SWAP during
Legalization, so existing backends should see no change in behaviour.
If they wish to deal with the enhanced node instead, they can call
setOperationAction on it. Beware: as a node with 2 results, it cannot
be selected from TableGen.
Currently, no use is made of the extra information provided in this
patch. Test updates are almost entirely adapting the input IR to the
new scheme.
Summary for out of tree users:
------------------------------
+ Legacy Bitcode files are upgraded during read.
+ Legacy assembly IR files will be invalid.
+ Front-ends must adapt to different type for "cmpxchg".
+ Backends should be unaffected by default.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@210903 91177308-0d34-0410-b5e6-96231b3b80d8
2014-06-13 14:24:07 +00:00
|
|
|
// Most backends expect to see the node which just returns the value loaded.
|
2015-01-07 21:27:10 +00:00
|
|
|
setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand);
|
IR: add "cmpxchg weak" variant to support permitted failure.
This commit adds a weak variant of the cmpxchg operation, as described
in C++11. A cmpxchg instruction with this modifier is permitted to
fail to store, even if the comparison indicated it should.
As a result, cmpxchg instructions must return a flag indicating
success in addition to their original iN value loaded. Thus, for
uniformity *all* cmpxchg instructions now return "{ iN, i1 }". The
second flag is 1 when the store succeeded.
At the DAG level, a new ATOMIC_CMP_SWAP_WITH_SUCCESS node has been
added as the natural representation for the new cmpxchg instructions.
It is a strong cmpxchg.
By default this gets Expanded to the existing ATOMIC_CMP_SWAP during
Legalization, so existing backends should see no change in behaviour.
If they wish to deal with the enhanced node instead, they can call
setOperationAction on it. Beware: as a node with 2 results, it cannot
be selected from TableGen.
Currently, no use is made of the extra information provided in this
patch. Test updates are almost entirely adapting the input IR to the
new scheme.
Summary for out of tree users:
------------------------------
+ Legacy Bitcode files are upgraded during read.
+ Legacy assembly IR files will be invalid.
+ Front-ends must adapt to different type for "cmpxchg".
+ Backends should be unaffected by default.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@210903 91177308-0d34-0410-b5e6-96231b3b80d8
2014-06-13 14:24:07 +00:00
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
// These operations default to expand.
|
2015-01-07 21:27:10 +00:00
|
|
|
setOperationAction(ISD::FGETSIGN, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::CONCAT_VECTORS, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FMINNUM, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FMAXNUM, VT, Expand);
|
2015-08-11 09:13:05 +00:00
|
|
|
setOperationAction(ISD::FMINNAN, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FMAXNAN, VT, Expand);
|
2015-02-20 22:10:33 +00:00
|
|
|
setOperationAction(ISD::FMAD, VT, Expand);
|
2015-05-15 09:03:15 +00:00
|
|
|
setOperationAction(ISD::SMIN, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::SMAX, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::UMIN, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::UMAX, VT, Expand);
|
2017-03-14 21:26:58 +00:00
|
|
|
setOperationAction(ISD::ABS, VT, Expand);
|
2013-08-09 04:13:44 +00:00
|
|
|
|
2015-04-29 16:30:46 +00:00
|
|
|
// Overflow operations default to expand
|
|
|
|
|
setOperationAction(ISD::SADDO, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::SSUBO, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::UADDO, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::USUBO, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::SMULO, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::UMULO, VT, Expand);
|
2015-12-11 23:11:52 +00:00
|
|
|
|
2017-04-30 19:24:09 +00:00
|
|
|
// ADDCARRY operations default to expand
|
|
|
|
|
setOperationAction(ISD::ADDCARRY, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::SUBCARRY, VT, Expand);
|
2017-06-01 11:14:17 +00:00
|
|
|
setOperationAction(ISD::SETCCCARRY, VT, Expand);
|
2017-04-30 19:24:09 +00:00
|
|
|
|
2016-04-28 03:34:31 +00:00
|
|
|
// These default to Expand so they will be expanded to CTLZ/CTTZ by default.
|
|
|
|
|
setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
|
|
|
|
|
|
2015-11-12 12:29:09 +00:00
|
|
|
setOperationAction(ISD::BITREVERSE, VT, Expand);
|
|
|
|
|
|
2013-08-09 04:13:44 +00:00
|
|
|
// These library functions default to expand.
|
2015-01-07 21:27:10 +00:00
|
|
|
setOperationAction(ISD::FROUND, VT, Expand);
|
2017-05-30 15:27:55 +00:00
|
|
|
setOperationAction(ISD::FPOWI, VT, Expand);
|
Add a llvm.copysign intrinsic
This adds a llvm.copysign intrinsic; We already have Libfunc recognition for
copysign (which is turned into the FCOPYSIGN SDAG node). In order to
autovectorize calls to copysign in the loop vectorizer, we need a corresponding
intrinsic as well.
In addition to the expected changes to the language reference, the loop
vectorizer, BasicTTI, and the SDAG builder (the intrinsic is transformed into
an FCOPYSIGN node, just like the function call), this also adds FCOPYSIGN to a
few lists in LegalizeVector{Ops,Types} so that vector copysigns can be
expanded.
In TargetLoweringBase::initActions, I've made the default action for FCOPYSIGN
be Expand for vector types. This seems correct for all in-tree targets, and I
think is the right thing to do because, previously, there was no way to generate
vector-values FCOPYSIGN nodes (and most targets don't specify an action for
vector-typed FCOPYSIGN).
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@188728 91177308-0d34-0410-b5e6-96231b3b80d8
2013-08-19 23:35:46 +00:00
|
|
|
|
|
|
|
|
// These operations default to expand for vector types.
|
2015-01-07 21:27:10 +00:00
|
|
|
if (VT.isVector()) {
|
|
|
|
|
setOperationAction(ISD::FCOPYSIGN, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand);
|
2014-07-09 22:53:04 +00:00
|
|
|
}
|
2015-12-01 11:40:55 +00:00
|
|
|
|
[stack-protection] Add support for MSVC buffer security check
Summary:
This patch is adding support for the MSVC buffer security check implementation
The buffer security check is turned on with the '/GS' compiler switch.
* https://msdn.microsoft.com/en-us/library/8dbf701c.aspx
* To be added to clang here: http://reviews.llvm.org/D20347
Some overview of buffer security check feature and implementation:
* https://msdn.microsoft.com/en-us/library/aa290051(VS.71).aspx
* http://www.ksyash.com/2011/01/buffer-overflow-protection-3/
* http://blog.osom.info/2012/02/understanding-vs-c-compilers-buffer.html
For the following example:
```
int example(int offset, int index) {
char buffer[10];
memset(buffer, 0xCC, index);
return buffer[index];
}
```
The MSVC compiler is adding these instructions to perform stack integrity check:
```
push ebp
mov ebp,esp
sub esp,50h
[1] mov eax,dword ptr [__security_cookie (01068024h)]
[2] xor eax,ebp
[3] mov dword ptr [ebp-4],eax
push ebx
push esi
push edi
mov eax,dword ptr [index]
push eax
push 0CCh
lea ecx,[buffer]
push ecx
call _memset (010610B9h)
add esp,0Ch
mov eax,dword ptr [index]
movsx eax,byte ptr buffer[eax]
pop edi
pop esi
pop ebx
[4] mov ecx,dword ptr [ebp-4]
[5] xor ecx,ebp
[6] call @__security_check_cookie@4 (01061276h)
mov esp,ebp
pop ebp
ret
```
The instrumentation above is:
* [1] is loading the global security canary,
* [3] is storing the local computed ([2]) canary to the guard slot,
* [4] is loading the guard slot and ([5]) re-compute the global canary,
* [6] is validating the resulting canary with the '__security_check_cookie' and performs error handling.
Overview of the current stack-protection implementation:
* lib/CodeGen/StackProtector.cpp
* There is a default stack-protection implementation applied on intermediate representation.
* The target can overload 'getIRStackGuard' method if it has a standard location for the stack protector cookie.
* An intrinsic 'Intrinsic::stackprotector' is added to the prologue. It will be expanded by the instruction selection pass (DAG or Fast).
* Basic Blocks are added to every instrumented function to receive the code for handling stack guard validation and errors handling.
* Guard manipulation and comparison are added directly to the intermediate representation.
* lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
* lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
* There is an implementation that adds instrumentation during instruction selection (for better handling of sibbling calls).
* see long comment above 'class StackProtectorDescriptor' declaration.
* The target needs to override 'getSDagStackGuard' to activate SDAG stack protection generation. (note: getIRStackGuard MUST be nullptr).
* 'getSDagStackGuard' returns the appropriate stack guard (security cookie)
* The code is generated by 'SelectionDAGBuilder.cpp' and 'SelectionDAGISel.cpp'.
* include/llvm/Target/TargetLowering.h
* Contains function to retrieve the default Guard 'Value'; should be overriden by each target to select which implementation is used and provide Guard 'Value'.
* lib/Target/X86/X86ISelLowering.cpp
* Contains the x86 specialisation; Guard 'Value' used by the SelectionDAG algorithm.
Function-based Instrumentation:
* The MSVC doesn't inline the stack guard comparison in every function. Instead, a call to '__security_check_cookie' is added to the epilogue before every return instructions.
* To support function-based instrumentation, this patch is
* adding a function to get the function-based check (llvm 'Value', see include/llvm/Target/TargetLowering.h),
* If provided, the stack protection instrumentation won't be inlined and a call to that function will be added to the prologue.
* modifying (SelectionDAGISel.cpp) do avoid producing basic blocks used for inline instrumentation,
* generating the function-based instrumentation during the ISEL pass (SelectionDAGBuilder.cpp),
* if FastISEL (not SelectionDAG), using the fallback which rely on the same function-based implemented over intermediate representation (StackProtector.cpp).
Modifications
* adding support for MSVC (lib/Target/X86/X86ISelLowering.cpp)
* adding support function-based instrumentation (lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp, .h)
Results
* IR generated instrumentation:
```
clang-cl /GS test.cc /Od /c -mllvm -print-isel-input
```
```
*** Final LLVM Code input to ISel ***
; Function Attrs: nounwind sspstrong
define i32 @"\01?example@@YAHHH@Z"(i32 %offset, i32 %index) #0 {
entry:
%StackGuardSlot = alloca i8* <<<-- Allocated guard slot
%0 = call i8* @llvm.stackguard() <<<-- Loading Stack Guard value
call void @llvm.stackprotector(i8* %0, i8** %StackGuardSlot) <<<-- Prologue intrinsic call (store to Guard slot)
%index.addr = alloca i32, align 4
%offset.addr = alloca i32, align 4
%buffer = alloca [10 x i8], align 1
store i32 %index, i32* %index.addr, align 4
store i32 %offset, i32* %offset.addr, align 4
%arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %buffer, i32 0, i32 0
%1 = load i32, i32* %index.addr, align 4
call void @llvm.memset.p0i8.i32(i8* %arraydecay, i8 -52, i32 %1, i32 1, i1 false)
%2 = load i32, i32* %index.addr, align 4
%arrayidx = getelementptr inbounds [10 x i8], [10 x i8]* %buffer, i32 0, i32 %2
%3 = load i8, i8* %arrayidx, align 1
%conv = sext i8 %3 to i32
%4 = load volatile i8*, i8** %StackGuardSlot <<<-- Loading Guard slot
call void @__security_check_cookie(i8* %4) <<<-- Epilogue function-based check
ret i32 %conv
}
```
* SelectionDAG generated instrumentation:
```
clang-cl /GS test.cc /O1 /c /FA
```
```
"?example@@YAHHH@Z": # @"\01?example@@YAHHH@Z"
# BB#0: # %entry
pushl %esi
subl $16, %esp
movl ___security_cookie, %eax <<<-- Loading Stack Guard value
movl 28(%esp), %esi
movl %eax, 12(%esp) <<<-- Store to Guard slot
leal 2(%esp), %eax
pushl %esi
pushl $204
pushl %eax
calll _memset
addl $12, %esp
movsbl 2(%esp,%esi), %esi
movl 12(%esp), %ecx <<<-- Loading Guard slot
calll @__security_check_cookie@4 <<<-- Epilogue function-based check
movl %esi, %eax
addl $16, %esp
popl %esi
retl
```
Reviewers: kcc, pcc, eugenis, rnk
Subscribers: majnemer, llvm-commits, hans, thakis, rnk
Differential Revision: http://reviews.llvm.org/D20346
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@272053 91177308-0d34-0410-b5e6-96231b3b80d8
2016-06-07 20:15:35 +00:00
|
|
|
// For most targets @llvm.get.dynamic.area.offset just returns 0.
|
2015-12-01 11:40:55 +00:00
|
|
|
setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand);
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Most targets ignore the @llvm.prefetch intrinsic.
|
|
|
|
|
setOperationAction(ISD::PREFETCH, MVT::Other, Expand);
|
|
|
|
|
|
2015-08-28 01:49:59 +00:00
|
|
|
// Most targets also ignore the @llvm.readcyclecounter intrinsic.
|
|
|
|
|
setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand);
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
// ConstantFP nodes default to expand. Targets can either change this to
|
|
|
|
|
// Legal, in which case all fp constants are legal, or use isFPImmLegal()
|
|
|
|
|
// to optimize expansions for certain constants.
|
|
|
|
|
setOperationAction(ISD::ConstantFP, MVT::f16, Expand);
|
|
|
|
|
setOperationAction(ISD::ConstantFP, MVT::f32, Expand);
|
|
|
|
|
setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
|
|
|
|
|
setOperationAction(ISD::ConstantFP, MVT::f80, Expand);
|
|
|
|
|
setOperationAction(ISD::ConstantFP, MVT::f128, Expand);
|
|
|
|
|
|
|
|
|
|
// These library functions default to expand.
|
2015-03-26 23:21:03 +00:00
|
|
|
for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
|
|
|
|
|
setOperationAction(ISD::FLOG , VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FLOG2, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FLOG10, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FEXP , VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FEXP2, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FFLOOR, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FNEARBYINT, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FCEIL, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FRINT, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FTRUNC, VT, Expand);
|
|
|
|
|
setOperationAction(ISD::FROUND, VT, Expand);
|
|
|
|
|
}
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
// Default ISD::TRAP to expand (which turns it into abort).
|
|
|
|
|
setOperationAction(ISD::TRAP, MVT::Other, Expand);
|
|
|
|
|
|
|
|
|
|
// On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
|
|
|
|
|
// here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
|
|
|
|
|
//
|
|
|
|
|
setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
|
|
|
|
|
}
|
|
|
|
|
|
2015-07-09 15:12:23 +00:00
|
|
|
MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
|
|
|
|
|
EVT) const {
|
2015-07-09 02:09:20 +00:00
|
|
|
return MVT::getIntegerVT(8 * DL.getPointerSize(0));
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2015-07-09 02:09:20 +00:00
|
|
|
EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy,
|
|
|
|
|
const DataLayout &DL) const {
|
2013-03-01 18:40:30 +00:00
|
|
|
assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
|
|
|
|
|
if (LHSTy.isVector())
|
|
|
|
|
return LHSTy;
|
2015-07-09 15:12:23 +00:00
|
|
|
return getScalarShiftAmountTy(DL, LHSTy);
|
2013-03-01 18:40:30 +00:00
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
|
|
|
|
|
assert(isTypeLegal(VT));
|
|
|
|
|
switch (Op) {
|
|
|
|
|
default:
|
|
|
|
|
return false;
|
|
|
|
|
case ISD::SDIV:
|
|
|
|
|
case ISD::UDIV:
|
|
|
|
|
case ISD::SREM:
|
|
|
|
|
case ISD::UREM:
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-07-01 18:10:20 +00:00
|
|
|
void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
|
|
|
|
|
// If the command-line option was specified, ignore this request.
|
|
|
|
|
if (!JumpIsExpensiveOverride.getNumOccurrences())
|
|
|
|
|
JumpIsExpensive = isExpensive;
|
|
|
|
|
}
|
|
|
|
|
|
2015-02-25 22:41:30 +00:00
|
|
|
TargetLoweringBase::LegalizeKind
|
|
|
|
|
TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
|
|
|
|
|
// If this is a simple type, use the ComputeRegisterProp mechanism.
|
|
|
|
|
if (VT.isSimple()) {
|
|
|
|
|
MVT SVT = VT.getSimpleVT();
|
|
|
|
|
assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType));
|
|
|
|
|
MVT NVT = TransformToType[SVT.SimpleTy];
|
|
|
|
|
LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
|
|
|
|
|
|
|
|
|
|
assert((LA == TypeLegal || LA == TypeSoftenFloat ||
|
|
|
|
|
ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger) &&
|
|
|
|
|
"Promote may not follow Expand or Promote");
|
|
|
|
|
|
|
|
|
|
if (LA == TypeSplitVector)
|
|
|
|
|
return LegalizeKind(LA,
|
|
|
|
|
EVT::getVectorVT(Context, SVT.getVectorElementType(),
|
|
|
|
|
SVT.getVectorNumElements() / 2));
|
|
|
|
|
if (LA == TypeScalarizeVector)
|
|
|
|
|
return LegalizeKind(LA, SVT.getVectorElementType());
|
|
|
|
|
return LegalizeKind(LA, NVT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Handle Extended Scalar Types.
|
|
|
|
|
if (!VT.isVector()) {
|
|
|
|
|
assert(VT.isInteger() && "Float types must be simple");
|
|
|
|
|
unsigned BitSize = VT.getSizeInBits();
|
|
|
|
|
// First promote to a power-of-two size, then expand if necessary.
|
|
|
|
|
if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
|
|
|
|
|
EVT NVT = VT.getRoundIntegerType(Context);
|
|
|
|
|
assert(NVT != VT && "Unable to round integer VT");
|
|
|
|
|
LegalizeKind NextStep = getTypeConversion(Context, NVT);
|
|
|
|
|
// Avoid multi-step promotion.
|
|
|
|
|
if (NextStep.first == TypePromoteInteger)
|
|
|
|
|
return NextStep;
|
|
|
|
|
// Return rounded integer type.
|
|
|
|
|
return LegalizeKind(TypePromoteInteger, NVT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return LegalizeKind(TypeExpandInteger,
|
|
|
|
|
EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Handle vector types.
|
|
|
|
|
unsigned NumElts = VT.getVectorNumElements();
|
|
|
|
|
EVT EltVT = VT.getVectorElementType();
|
|
|
|
|
|
|
|
|
|
// Vectors with only one element are always scalarized.
|
|
|
|
|
if (NumElts == 1)
|
|
|
|
|
return LegalizeKind(TypeScalarizeVector, EltVT);
|
|
|
|
|
|
|
|
|
|
// Try to widen vector elements until the element type is a power of two and
|
|
|
|
|
// promote it to a legal type later on, for example:
|
|
|
|
|
// <3 x i8> -> <4 x i8> -> <4 x i32>
|
|
|
|
|
if (EltVT.isInteger()) {
|
|
|
|
|
// Vectors with a number of elements that is not a power of two are always
|
|
|
|
|
// widened, for example <3 x i8> -> <4 x i8>.
|
|
|
|
|
if (!VT.isPow2VectorType()) {
|
|
|
|
|
NumElts = (unsigned)NextPowerOf2(NumElts);
|
|
|
|
|
EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
|
|
|
|
|
return LegalizeKind(TypeWidenVector, NVT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Examine the element type.
|
|
|
|
|
LegalizeKind LK = getTypeConversion(Context, EltVT);
|
|
|
|
|
|
|
|
|
|
// If type is to be expanded, split the vector.
|
|
|
|
|
// <4 x i140> -> <2 x i140>
|
|
|
|
|
if (LK.first == TypeExpandInteger)
|
|
|
|
|
return LegalizeKind(TypeSplitVector,
|
|
|
|
|
EVT::getVectorVT(Context, EltVT, NumElts / 2));
|
|
|
|
|
|
|
|
|
|
// Promote the integer element types until a legal vector type is found
|
|
|
|
|
// or until the element integer type is too big. If a legal type was not
|
|
|
|
|
// found, fallback to the usual mechanism of widening/splitting the
|
|
|
|
|
// vector.
|
|
|
|
|
EVT OldEltVT = EltVT;
|
|
|
|
|
while (1) {
|
|
|
|
|
// Increase the bitwidth of the element to the next pow-of-two
|
|
|
|
|
// (which is greater than 8 bits).
|
|
|
|
|
EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
|
|
|
|
|
.getRoundIntegerType(Context);
|
|
|
|
|
|
|
|
|
|
// Stop trying when getting a non-simple element type.
|
|
|
|
|
// Note that vector elements may be greater than legal vector element
|
|
|
|
|
// types. Example: X86 XMM registers hold 64bit element on 32bit
|
|
|
|
|
// systems.
|
|
|
|
|
if (!EltVT.isSimple())
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
// Build a new vector type and check if it is legal.
|
|
|
|
|
MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
|
|
|
|
|
// Found a legal promoted vector type.
|
|
|
|
|
if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
|
|
|
|
|
return LegalizeKind(TypePromoteInteger,
|
|
|
|
|
EVT::getVectorVT(Context, EltVT, NumElts));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Reset the type to the unexpanded type if we did not find a legal vector
|
|
|
|
|
// type with a promoted vector element type.
|
|
|
|
|
EltVT = OldEltVT;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Try to widen the vector until a legal type is found.
|
|
|
|
|
// If there is no wider legal type, split the vector.
|
|
|
|
|
while (1) {
|
|
|
|
|
// Round up to the next power of 2.
|
|
|
|
|
NumElts = (unsigned)NextPowerOf2(NumElts);
|
|
|
|
|
|
|
|
|
|
// If there is no simple vector type with this many elements then there
|
|
|
|
|
// cannot be a larger legal vector type. Note that this assumes that
|
|
|
|
|
// there are no skipped intermediate vector types in the simple types.
|
|
|
|
|
if (!EltVT.isSimple())
|
|
|
|
|
break;
|
|
|
|
|
MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
|
|
|
|
|
if (LargerVector == MVT())
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
// If this type is legal then widen the vector.
|
|
|
|
|
if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
|
|
|
|
|
return LegalizeKind(TypeWidenVector, LargerVector);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Widen odd vectors to next power of two.
|
|
|
|
|
if (!VT.isPow2VectorType()) {
|
|
|
|
|
EVT NVT = VT.getPow2VectorType(Context);
|
|
|
|
|
return LegalizeKind(TypeWidenVector, NVT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Vectors with illegal element types are expanded.
|
|
|
|
|
EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorNumElements() / 2);
|
|
|
|
|
return LegalizeKind(TypeSplitVector, NVT);
|
|
|
|
|
}
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
|
|
|
|
|
unsigned &NumIntermediates,
|
|
|
|
|
MVT &RegisterVT,
|
|
|
|
|
TargetLoweringBase *TLI) {
|
|
|
|
|
// Figure out the right, legal destination reg to copy into.
|
|
|
|
|
unsigned NumElts = VT.getVectorNumElements();
|
|
|
|
|
MVT EltTy = VT.getVectorElementType();
|
|
|
|
|
|
|
|
|
|
unsigned NumVectorRegs = 1;
|
|
|
|
|
|
|
|
|
|
// FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we
|
|
|
|
|
// could break down into LHS/RHS like LegalizeDAG does.
|
|
|
|
|
if (!isPowerOf2_32(NumElts)) {
|
|
|
|
|
NumVectorRegs = NumElts;
|
|
|
|
|
NumElts = 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Divide the input until we get to a supported size. This will always
|
|
|
|
|
// end with a scalar if the target doesn't support vectors.
|
|
|
|
|
while (NumElts > 1 && !TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))) {
|
|
|
|
|
NumElts >>= 1;
|
|
|
|
|
NumVectorRegs <<= 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
NumIntermediates = NumVectorRegs;
|
|
|
|
|
|
|
|
|
|
MVT NewVT = MVT::getVectorVT(EltTy, NumElts);
|
|
|
|
|
if (!TLI->isTypeLegal(NewVT))
|
|
|
|
|
NewVT = EltTy;
|
|
|
|
|
IntermediateVT = NewVT;
|
|
|
|
|
|
|
|
|
|
unsigned NewVTSize = NewVT.getSizeInBits();
|
|
|
|
|
|
|
|
|
|
// Convert sizes such as i33 to i64.
|
|
|
|
|
if (!isPowerOf2_32(NewVTSize))
|
|
|
|
|
NewVTSize = NextPowerOf2(NewVTSize);
|
|
|
|
|
|
|
|
|
|
MVT DestVT = TLI->getRegisterType(NewVT);
|
|
|
|
|
RegisterVT = DestVT;
|
|
|
|
|
if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
|
|
|
|
|
return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
|
|
|
|
|
|
|
|
|
|
// Otherwise, promotion or legal types use the same number of registers as
|
|
|
|
|
// the vector decimated to the appropriate level.
|
|
|
|
|
return NumVectorRegs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// isLegalRC - Return true if the value types that can be represented by the
|
|
|
|
|
/// specified register class are all legal.
|
2017-04-24 19:51:12 +00:00
|
|
|
bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI,
|
|
|
|
|
const TargetRegisterClass &RC) const {
|
|
|
|
|
for (auto I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
|
2013-01-11 20:05:37 +00:00
|
|
|
if (isTypeLegal(*I))
|
|
|
|
|
return true;
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2013-11-29 03:07:54 +00:00
|
|
|
/// Replace/modify any TargetFrameIndex operands with a targte-dependent
|
|
|
|
|
/// sequence of memory operands that is recognized by PrologEpilogInserter.
|
2016-06-30 22:52:52 +00:00
|
|
|
MachineBasicBlock *
|
|
|
|
|
TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
|
2013-11-29 03:07:54 +00:00
|
|
|
MachineBasicBlock *MBB) const {
|
2016-06-30 22:52:52 +00:00
|
|
|
MachineInstr *MI = &InitialMI;
|
2013-11-29 03:07:54 +00:00
|
|
|
MachineFunction &MF = *MI->getParent()->getParent();
|
2016-07-28 18:40:00 +00:00
|
|
|
MachineFrameInfo &MFI = MF.getFrameInfo();
|
2015-12-23 23:44:28 +00:00
|
|
|
|
|
|
|
|
// We're handling multiple types of operands here:
|
|
|
|
|
// PATCHPOINT MetaArgs - live-in, read only, direct
|
|
|
|
|
// STATEPOINT Deopt Spill - live-through, read only, indirect
|
|
|
|
|
// STATEPOINT Deopt Alloca - live-through, read only, direct
|
|
|
|
|
// (We're currently conservative and mark the deopt slots read/write in
|
|
|
|
|
// practice.)
|
|
|
|
|
// STATEPOINT GC Spill - live-through, read/write, indirect
|
|
|
|
|
// STATEPOINT GC Alloca - live-through, read/write, direct
|
|
|
|
|
// The live-in vs live-through is handled already (the live through ones are
|
|
|
|
|
// all stack slots), but we need to handle the different type of stackmap
|
|
|
|
|
// operands and memory effects here.
|
2013-11-29 03:07:54 +00:00
|
|
|
|
|
|
|
|
// MI changes inside this loop as we grow operands.
|
|
|
|
|
for(unsigned OperIdx = 0; OperIdx != MI->getNumOperands(); ++OperIdx) {
|
|
|
|
|
MachineOperand &MO = MI->getOperand(OperIdx);
|
|
|
|
|
if (!MO.isFI())
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
// foldMemoryOperand builds a new MI after replacing a single FI operand
|
|
|
|
|
// with the canonical set of five x86 addressing-mode operands.
|
|
|
|
|
int FI = MO.getIndex();
|
|
|
|
|
MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
|
|
|
|
|
|
|
|
|
|
// Copy operands before the frame-index.
|
|
|
|
|
for (unsigned i = 0; i < OperIdx; ++i)
|
2017-01-13 09:58:52 +00:00
|
|
|
MIB.add(MI->getOperand(i));
|
2015-12-23 23:44:28 +00:00
|
|
|
// Add frame index operands recognized by stackmaps.cpp
|
|
|
|
|
if (MFI.isStatepointSpillSlotObjectIndex(FI)) {
|
|
|
|
|
// indirect-mem-ref tag, size, #FI, offset.
|
|
|
|
|
// Used for spills inserted by StatepointLowering. This codepath is not
|
|
|
|
|
// used for patchpoints/stackmaps at all, for these spilling is done via
|
|
|
|
|
// foldMemoryOperand callback only.
|
|
|
|
|
assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
|
|
|
|
|
MIB.addImm(StackMaps::IndirectMemRefOp);
|
|
|
|
|
MIB.addImm(MFI.getObjectSize(FI));
|
2017-01-13 09:58:52 +00:00
|
|
|
MIB.add(MI->getOperand(OperIdx));
|
2015-12-23 23:44:28 +00:00
|
|
|
MIB.addImm(0);
|
|
|
|
|
} else {
|
|
|
|
|
// direct-mem-ref tag, #FI, offset.
|
|
|
|
|
// Used by patchpoint, and direct alloca arguments to statepoints
|
|
|
|
|
MIB.addImm(StackMaps::DirectMemRefOp);
|
2017-01-13 09:58:52 +00:00
|
|
|
MIB.add(MI->getOperand(OperIdx));
|
2015-12-23 23:44:28 +00:00
|
|
|
MIB.addImm(0);
|
|
|
|
|
}
|
2013-11-29 03:07:54 +00:00
|
|
|
// Copy the operands after the frame index.
|
|
|
|
|
for (unsigned i = OperIdx + 1; i != MI->getNumOperands(); ++i)
|
2017-01-13 09:58:52 +00:00
|
|
|
MIB.add(MI->getOperand(i));
|
2013-11-29 03:07:54 +00:00
|
|
|
|
|
|
|
|
// Inherit previous memory operands.
|
|
|
|
|
MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
|
|
|
|
|
assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
|
|
|
|
|
|
|
|
|
|
// Add a new memory operand for this FI.
|
|
|
|
|
assert(MFI.getObjectOffset(FI) != -1);
|
2014-12-01 22:52:56 +00:00
|
|
|
|
2016-07-15 18:26:59 +00:00
|
|
|
auto Flags = MachineMemOperand::MOLoad;
|
2014-12-01 22:52:56 +00:00
|
|
|
if (MI->getOpcode() == TargetOpcode::STATEPOINT) {
|
|
|
|
|
Flags |= MachineMemOperand::MOStore;
|
|
|
|
|
Flags |= MachineMemOperand::MOVolatile;
|
|
|
|
|
}
|
2014-08-04 21:25:23 +00:00
|
|
|
MachineMemOperand *MMO = MF.getMachineMemOperand(
|
2015-08-11 23:09:45 +00:00
|
|
|
MachinePointerInfo::getFixedStack(MF, FI), Flags,
|
2015-07-16 06:11:10 +00:00
|
|
|
MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI));
|
2013-11-29 03:07:54 +00:00
|
|
|
MIB->addMemOperand(MF, MMO);
|
|
|
|
|
|
|
|
|
|
// Replace the instruction and update the operand index.
|
|
|
|
|
MBB->insert(MachineBasicBlock::iterator(MI), MIB);
|
|
|
|
|
OperIdx += (MIB->getNumOperands() - MI->getNumOperands()) - 1;
|
|
|
|
|
MI->eraseFromParent();
|
|
|
|
|
MI = MIB;
|
|
|
|
|
}
|
|
|
|
|
return MBB;
|
|
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
/// findRepresentativeClass - Return the largest legal super-reg register class
|
|
|
|
|
/// of the register class for the specified type and its associated "cost".
|
2015-03-03 19:47:14 +00:00
|
|
|
// This function is in TargetLowering because it uses RegClassForVT which would
|
|
|
|
|
// need to be moved to TargetRegisterInfo and would necessitate moving
|
|
|
|
|
// isTypeLegal over as well - a massive change that would just require
|
|
|
|
|
// TargetLowering having a TargetRegisterInfo class member that it would use.
|
2015-02-26 00:00:24 +00:00
|
|
|
std::pair<const TargetRegisterClass *, uint8_t>
|
|
|
|
|
TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI,
|
|
|
|
|
MVT VT) const {
|
2013-01-11 20:05:37 +00:00
|
|
|
const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
|
|
|
|
|
if (!RC)
|
|
|
|
|
return std::make_pair(RC, 0);
|
|
|
|
|
|
|
|
|
|
// Compute the set of all super-register classes.
|
|
|
|
|
BitVector SuperRegRC(TRI->getNumRegClasses());
|
|
|
|
|
for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
|
|
|
|
|
SuperRegRC.setBitsInMask(RCI.getMask());
|
|
|
|
|
|
|
|
|
|
// Find the first legal register class with the largest spill size.
|
|
|
|
|
const TargetRegisterClass *BestRC = RC;
|
2017-05-17 01:07:53 +00:00
|
|
|
for (unsigned i : SuperRegRC.set_bits()) {
|
2013-01-11 20:05:37 +00:00
|
|
|
const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
|
|
|
|
|
// We want the largest possible spill size.
|
2017-04-24 18:55:33 +00:00
|
|
|
if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
|
2013-01-11 20:05:37 +00:00
|
|
|
continue;
|
2017-04-24 19:51:12 +00:00
|
|
|
if (!isLegalRC(*TRI, *SuperRC))
|
2013-01-11 20:05:37 +00:00
|
|
|
continue;
|
|
|
|
|
BestRC = SuperRC;
|
|
|
|
|
}
|
|
|
|
|
return std::make_pair(BestRC, 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// computeRegisterProperties - Once all of the register classes are added,
|
|
|
|
|
/// this allows us to compute derived properties we expose.
|
2015-02-26 00:00:24 +00:00
|
|
|
void TargetLoweringBase::computeRegisterProperties(
|
|
|
|
|
const TargetRegisterInfo *TRI) {
|
2014-11-17 00:26:50 +00:00
|
|
|
static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE,
|
|
|
|
|
"Too many value types for ValueTypeActions to hold!");
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
// Everything defaults to needing one register.
|
|
|
|
|
for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) {
|
|
|
|
|
NumRegistersForVT[i] = 1;
|
|
|
|
|
RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
|
|
|
|
|
}
|
|
|
|
|
// ...except isVoid, which doesn't need any registers.
|
|
|
|
|
NumRegistersForVT[MVT::isVoid] = 0;
|
|
|
|
|
|
|
|
|
|
// Find the largest integer register class.
|
|
|
|
|
unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
|
2014-04-14 00:51:57 +00:00
|
|
|
for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
|
2013-01-11 20:05:37 +00:00
|
|
|
assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
|
|
|
|
|
|
|
|
|
|
// Every integer value type larger than this largest register takes twice as
|
|
|
|
|
// many registers to represent as the previous ValueType.
|
|
|
|
|
for (unsigned ExpandedReg = LargestIntReg + 1;
|
|
|
|
|
ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
|
|
|
|
|
NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
|
|
|
|
|
RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
|
|
|
|
|
TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
|
|
|
|
|
ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
|
|
|
|
|
TypeExpandInteger);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Inspect all of the ValueType's smaller than the largest integer
|
|
|
|
|
// register to see which ones need promotion.
|
|
|
|
|
unsigned LegalIntReg = LargestIntReg;
|
|
|
|
|
for (unsigned IntReg = LargestIntReg - 1;
|
|
|
|
|
IntReg >= (unsigned)MVT::i1; --IntReg) {
|
|
|
|
|
MVT IVT = (MVT::SimpleValueType)IntReg;
|
|
|
|
|
if (isTypeLegal(IVT)) {
|
|
|
|
|
LegalIntReg = IntReg;
|
|
|
|
|
} else {
|
|
|
|
|
RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
|
|
|
|
|
(const MVT::SimpleValueType)LegalIntReg;
|
|
|
|
|
ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ppcf128 type is really two f64's.
|
|
|
|
|
if (!isTypeLegal(MVT::ppcf128)) {
|
2016-02-04 14:43:50 +00:00
|
|
|
if (isTypeLegal(MVT::f64)) {
|
|
|
|
|
NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
|
|
|
|
|
RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
|
|
|
|
|
TransformToType[MVT::ppcf128] = MVT::f64;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
|
|
|
|
|
} else {
|
|
|
|
|
NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
|
|
|
|
|
RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
|
|
|
|
|
TransformToType[MVT::ppcf128] = MVT::i128;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
|
|
|
|
|
}
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2013-03-01 21:11:44 +00:00
|
|
|
// Decide how to handle f128. If the target does not have native f128 support,
|
|
|
|
|
// expand it to i128 and we will be generating soft float library calls.
|
|
|
|
|
if (!isTypeLegal(MVT::f128)) {
|
|
|
|
|
NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
|
|
|
|
|
RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
|
|
|
|
|
TransformToType[MVT::f128] = MVT::i128;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
|
|
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
// Decide how to handle f64. If the target does not have native f64 support,
|
|
|
|
|
// expand it to i64 and we will be generating soft float library calls.
|
|
|
|
|
if (!isTypeLegal(MVT::f64)) {
|
|
|
|
|
NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
|
|
|
|
|
RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
|
|
|
|
|
TransformToType[MVT::f64] = MVT::i64;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
|
|
|
|
|
}
|
|
|
|
|
|
[CodeGen] "PromoteInteger" f32 to f64 doesn't make sense.
The original f32->f64 promotion logic was refactored into roughly the
currently shape in r37781. However, starting with r132263, the
legalizer has been split into different kinds, and the previous
"Promote" (which did the right thing) was search-and-replace'd into
"PromoteInteger". The divide gradually deepened, with type legalization
("PromoteInteger") being separated from ops legalization
("Promote", which still works for floating point ops).
Fast-forward to today: there's no in-tree target with legal f64 but
illegal f32 (rather: no tests were harmed in the making of this patch).
With such a target, i.e., if you trick the legalizer into going through
the PromoteInteger path for FP, you get the expected brokenness.
For instance, there's no PromoteIntRes_FADD (the name itself sounds
wrong), so we'll just hit some assert in the PromoteInteger path.
Don't pretend we can promote f32 to f64. Instead, always soften.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@233464 91177308-0d34-0410-b5e6-96231b3b80d8
2015-03-28 01:22:37 +00:00
|
|
|
// Decide how to handle f32. If the target does not have native f32 support,
|
|
|
|
|
// expand it to i32 and we will be generating soft float library calls.
|
2013-01-11 20:05:37 +00:00
|
|
|
if (!isTypeLegal(MVT::f32)) {
|
[CodeGen] "PromoteInteger" f32 to f64 doesn't make sense.
The original f32->f64 promotion logic was refactored into roughly the
currently shape in r37781. However, starting with r132263, the
legalizer has been split into different kinds, and the previous
"Promote" (which did the right thing) was search-and-replace'd into
"PromoteInteger". The divide gradually deepened, with type legalization
("PromoteInteger") being separated from ops legalization
("Promote", which still works for floating point ops).
Fast-forward to today: there's no in-tree target with legal f64 but
illegal f32 (rather: no tests were harmed in the making of this patch).
With such a target, i.e., if you trick the legalizer into going through
the PromoteInteger path for FP, you get the expected brokenness.
For instance, there's no PromoteIntRes_FADD (the name itself sounds
wrong), so we'll just hit some assert in the PromoteInteger path.
Don't pretend we can promote f32 to f64. Instead, always soften.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@233464 91177308-0d34-0410-b5e6-96231b3b80d8
2015-03-28 01:22:37 +00:00
|
|
|
NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
|
|
|
|
|
RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
|
|
|
|
|
TransformToType[MVT::f32] = MVT::i32;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2015-11-09 11:03:18 +00:00
|
|
|
// Decide how to handle f16. If the target does not have native f16 support,
|
|
|
|
|
// promote it to f32, because there are no f16 library calls (except for
|
|
|
|
|
// conversions).
|
2014-07-18 12:41:46 +00:00
|
|
|
if (!isTypeLegal(MVT::f16)) {
|
2015-11-09 11:03:18 +00:00
|
|
|
NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
|
|
|
|
|
RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
|
|
|
|
|
TransformToType[MVT::f16] = MVT::f32;
|
|
|
|
|
ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
|
2014-07-18 12:41:46 +00:00
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
// Loop over all of the vector value types to see which need transformations.
|
|
|
|
|
for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
|
|
|
|
|
i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
|
2014-07-03 00:23:43 +00:00
|
|
|
MVT VT = (MVT::SimpleValueType) i;
|
|
|
|
|
if (isTypeLegal(VT))
|
|
|
|
|
continue;
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
MVT EltVT = VT.getVectorElementType();
|
|
|
|
|
unsigned NElts = VT.getVectorNumElements();
|
2014-07-03 00:23:43 +00:00
|
|
|
bool IsLegalWiderType = false;
|
|
|
|
|
LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
|
|
|
|
|
switch (PreferredAction) {
|
|
|
|
|
case TypePromoteInteger: {
|
|
|
|
|
// Try to promote the elements of integer vectors. If no legal
|
|
|
|
|
// promotion was found, fall through to the widen-vector method.
|
2016-04-22 21:16:17 +00:00
|
|
|
for (unsigned nVT = i + 1; nVT <= MVT::LAST_INTEGER_VECTOR_VALUETYPE; ++nVT) {
|
2014-07-03 00:23:43 +00:00
|
|
|
MVT SVT = (MVT::SimpleValueType) nVT;
|
2013-01-11 20:05:37 +00:00
|
|
|
// Promote vectors of integers to vectors with the same number
|
|
|
|
|
// of elements, with a wider element type.
|
2016-09-14 16:37:15 +00:00
|
|
|
if (SVT.getScalarSizeInBits() > EltVT.getSizeInBits() &&
|
2016-04-22 21:16:17 +00:00
|
|
|
SVT.getVectorNumElements() == NElts && isTypeLegal(SVT)) {
|
2013-01-11 20:05:37 +00:00
|
|
|
TransformToType[i] = SVT;
|
|
|
|
|
RegisterTypeForVT[i] = SVT;
|
|
|
|
|
NumRegistersForVT[i] = 1;
|
|
|
|
|
ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
|
|
|
|
|
IsLegalWiderType = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-07-03 00:23:43 +00:00
|
|
|
if (IsLegalWiderType)
|
|
|
|
|
break;
|
2017-06-03 05:11:14 +00:00
|
|
|
LLVM_FALLTHROUGH;
|
2014-07-03 00:23:43 +00:00
|
|
|
}
|
|
|
|
|
case TypeWidenVector: {
|
2013-01-11 20:05:37 +00:00
|
|
|
// Try to widen the vector.
|
2014-07-03 00:23:43 +00:00
|
|
|
for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
|
|
|
|
|
MVT SVT = (MVT::SimpleValueType) nVT;
|
|
|
|
|
if (SVT.getVectorElementType() == EltVT
|
|
|
|
|
&& SVT.getVectorNumElements() > NElts && isTypeLegal(SVT)) {
|
2013-01-11 20:05:37 +00:00
|
|
|
TransformToType[i] = SVT;
|
|
|
|
|
RegisterTypeForVT[i] = SVT;
|
|
|
|
|
NumRegistersForVT[i] = 1;
|
|
|
|
|
ValueTypeActions.setTypeAction(VT, TypeWidenVector);
|
|
|
|
|
IsLegalWiderType = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-07-03 00:23:43 +00:00
|
|
|
if (IsLegalWiderType)
|
|
|
|
|
break;
|
2017-06-03 05:11:14 +00:00
|
|
|
LLVM_FALLTHROUGH;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
2014-07-03 00:23:43 +00:00
|
|
|
case TypeSplitVector:
|
|
|
|
|
case TypeScalarizeVector: {
|
|
|
|
|
MVT IntermediateVT;
|
|
|
|
|
MVT RegisterVT;
|
|
|
|
|
unsigned NumIntermediates;
|
|
|
|
|
NumRegistersForVT[i] = getVectorTypeBreakdownMVT(VT, IntermediateVT,
|
|
|
|
|
NumIntermediates, RegisterVT, this);
|
|
|
|
|
RegisterTypeForVT[i] = RegisterVT;
|
|
|
|
|
|
|
|
|
|
MVT NVT = VT.getPow2VectorType();
|
|
|
|
|
if (NVT == VT) {
|
|
|
|
|
// Type is already a power of 2. The default action is to split.
|
|
|
|
|
TransformToType[i] = MVT::Other;
|
|
|
|
|
if (PreferredAction == TypeScalarizeVector)
|
|
|
|
|
ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
|
2014-10-31 02:35:34 +00:00
|
|
|
else if (PreferredAction == TypeSplitVector)
|
2014-07-03 00:23:43 +00:00
|
|
|
ValueTypeActions.setTypeAction(VT, TypeSplitVector);
|
2014-10-31 02:35:34 +00:00
|
|
|
else
|
|
|
|
|
// Set type action according to the number of elements.
|
|
|
|
|
ValueTypeActions.setTypeAction(VT, NElts == 1 ? TypeScalarizeVector
|
|
|
|
|
: TypeSplitVector);
|
2014-07-03 00:23:43 +00:00
|
|
|
} else {
|
|
|
|
|
TransformToType[i] = NVT;
|
|
|
|
|
ValueTypeActions.setTypeAction(VT, TypeWidenVector);
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
llvm_unreachable("Unknown vector legalization action!");
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Determine the 'representative' register class for each value type.
|
|
|
|
|
// An representative register class is the largest (meaning one which is
|
|
|
|
|
// not a sub-register class / subreg register class) legal register class for
|
|
|
|
|
// a group of value types. For example, on i386, i8, i16, and i32
|
|
|
|
|
// representative would be GR32; while on x86_64 it's GR64.
|
|
|
|
|
for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) {
|
|
|
|
|
const TargetRegisterClass* RRC;
|
|
|
|
|
uint8_t Cost;
|
2015-02-26 00:00:24 +00:00
|
|
|
std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i);
|
2013-01-11 20:05:37 +00:00
|
|
|
RepRegClassForVT[i] = RRC;
|
|
|
|
|
RepRegClassCostForVT[i] = Cost;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2015-07-09 02:09:04 +00:00
|
|
|
EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
|
|
|
|
|
EVT VT) const {
|
2013-01-11 20:05:37 +00:00
|
|
|
assert(!VT.isVector() && "No default SetCC type for vectors!");
|
2015-07-09 02:09:04 +00:00
|
|
|
return getPointerTy(DL).SimpleTy;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const {
|
|
|
|
|
return MVT::i32; // return the default value
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getVectorTypeBreakdown - Vector types are broken down into some number of
|
|
|
|
|
/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
|
|
|
|
|
/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
|
|
|
|
|
/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
|
|
|
|
|
///
|
|
|
|
|
/// This method returns the number of registers needed, and the VT for each
|
|
|
|
|
/// register. It also returns the VT and quantity of the intermediate values
|
|
|
|
|
/// before they are promoted/expanded.
|
|
|
|
|
///
|
|
|
|
|
unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
|
|
|
|
|
EVT &IntermediateVT,
|
|
|
|
|
unsigned &NumIntermediates,
|
|
|
|
|
MVT &RegisterVT) const {
|
|
|
|
|
unsigned NumElts = VT.getVectorNumElements();
|
|
|
|
|
|
|
|
|
|
// If there is a wider vector type with the same element type as this one,
|
|
|
|
|
// or a promoted vector type that has the same number of elements which
|
|
|
|
|
// are wider, then we should convert to that legal vector type.
|
|
|
|
|
// This handles things like <2 x float> -> <4 x float> and
|
|
|
|
|
// <4 x i1> -> <4 x i32>.
|
|
|
|
|
LegalizeTypeAction TA = getTypeAction(Context, VT);
|
|
|
|
|
if (NumElts != 1 && (TA == TypeWidenVector || TA == TypePromoteInteger)) {
|
|
|
|
|
EVT RegisterEVT = getTypeToTransformTo(Context, VT);
|
|
|
|
|
if (isTypeLegal(RegisterEVT)) {
|
|
|
|
|
IntermediateVT = RegisterEVT;
|
|
|
|
|
RegisterVT = RegisterEVT.getSimpleVT();
|
|
|
|
|
NumIntermediates = 1;
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Figure out the right, legal destination reg to copy into.
|
|
|
|
|
EVT EltTy = VT.getVectorElementType();
|
|
|
|
|
|
|
|
|
|
unsigned NumVectorRegs = 1;
|
|
|
|
|
|
|
|
|
|
// FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we
|
|
|
|
|
// could break down into LHS/RHS like LegalizeDAG does.
|
|
|
|
|
if (!isPowerOf2_32(NumElts)) {
|
|
|
|
|
NumVectorRegs = NumElts;
|
|
|
|
|
NumElts = 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Divide the input until we get to a supported size. This will always
|
|
|
|
|
// end with a scalar if the target doesn't support vectors.
|
|
|
|
|
while (NumElts > 1 && !isTypeLegal(
|
|
|
|
|
EVT::getVectorVT(Context, EltTy, NumElts))) {
|
|
|
|
|
NumElts >>= 1;
|
|
|
|
|
NumVectorRegs <<= 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
NumIntermediates = NumVectorRegs;
|
|
|
|
|
|
|
|
|
|
EVT NewVT = EVT::getVectorVT(Context, EltTy, NumElts);
|
|
|
|
|
if (!isTypeLegal(NewVT))
|
|
|
|
|
NewVT = EltTy;
|
|
|
|
|
IntermediateVT = NewVT;
|
|
|
|
|
|
|
|
|
|
MVT DestVT = getRegisterType(Context, NewVT);
|
|
|
|
|
RegisterVT = DestVT;
|
|
|
|
|
unsigned NewVTSize = NewVT.getSizeInBits();
|
|
|
|
|
|
|
|
|
|
// Convert sizes such as i33 to i64.
|
|
|
|
|
if (!isPowerOf2_32(NewVTSize))
|
|
|
|
|
NewVTSize = NextPowerOf2(NewVTSize);
|
|
|
|
|
|
|
|
|
|
if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
|
|
|
|
|
return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
|
|
|
|
|
|
|
|
|
|
// Otherwise, promotion or legal types use the same number of registers as
|
|
|
|
|
// the vector decimated to the appropriate level.
|
|
|
|
|
return NumVectorRegs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Get the EVTs and ArgFlags collections that represent the legalized return
|
|
|
|
|
/// type of the given function. This does not require a DAG or a return value,
|
|
|
|
|
/// and is suitable for use before any DAGs for the function are constructed.
|
|
|
|
|
/// TODO: Move this out of TargetLowering.cpp.
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
void llvm::GetReturnInfo(Type *ReturnType, AttributeList attr,
|
2013-01-11 20:05:37 +00:00
|
|
|
SmallVectorImpl<ISD::OutputArg> &Outs,
|
2015-07-09 01:57:34 +00:00
|
|
|
const TargetLowering &TLI, const DataLayout &DL) {
|
2013-01-11 20:05:37 +00:00
|
|
|
SmallVector<EVT, 4> ValueVTs;
|
2015-07-09 01:57:34 +00:00
|
|
|
ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
|
2013-01-11 20:05:37 +00:00
|
|
|
unsigned NumValues = ValueVTs.size();
|
|
|
|
|
if (NumValues == 0) return;
|
|
|
|
|
|
|
|
|
|
for (unsigned j = 0, f = NumValues; j != f; ++j) {
|
|
|
|
|
EVT VT = ValueVTs[j];
|
|
|
|
|
ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
|
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
|
2013-01-11 20:05:37 +00:00
|
|
|
ExtendKind = ISD::SIGN_EXTEND;
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt))
|
2013-01-11 20:05:37 +00:00
|
|
|
ExtendKind = ISD::ZERO_EXTEND;
|
|
|
|
|
|
|
|
|
|
// FIXME: C calling convention requires the return type to be promoted to
|
|
|
|
|
// at least 32-bit. But this is not necessary for non-C calling
|
|
|
|
|
// conventions. The frontend should mark functions whose return values
|
|
|
|
|
// require promoting with signext or zeroext attributes.
|
|
|
|
|
if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) {
|
|
|
|
|
MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32);
|
|
|
|
|
if (VT.bitsLT(MinVT))
|
|
|
|
|
VT = MinVT;
|
|
|
|
|
}
|
|
|
|
|
|
Reland "[SelectionDAG] Enable target specific vector scalarization of calls and returns"
By target hookifying getRegisterType, getNumRegisters, getVectorBreakdown,
backends can request that LLVM to scalarize vector types for calls
and returns.
The MIPS vector ABI requires that vector arguments and returns are passed in
integer registers. With SelectionDAG's new hooks, the MIPS backend can now
handle LLVM-IR with vector types in calls and returns. E.g.
'call @foo(<4 x i32> %4)'.
Previously these cases would be scalarized for the MIPS O32/N32/N64 ABI for
calls and returns if vector types were not legal. If vector types were legal,
a single 128bit vector argument would be assigned to a single 32 bit / 64 bit
integer register.
By teaching the MIPS backend to inspect the original types, it can now
implement the MIPS vector ABI which requires a particular method of
scalarizing vectors.
Previously, the MIPS backend relied on clang to scalarize types such as "call
@foo(<4 x float> %a) into "call @foo(i32 inreg %1, i32 inreg %2, i32 inreg %3,
i32 inreg %4)".
This patch enables the MIPS backend to take either form for vector types.
The previous version of this patch had a "conditional move or jump depends on
uninitialized value".
Reviewers: zoran.jovanovic, jaydeep, vkalintiris, slthakur
Differential Revision: https://reviews.llvm.org/D27845
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@305083 91177308-0d34-0410-b5e6-96231b3b80d8
2017-06-09 14:37:08 +00:00
|
|
|
unsigned NumParts =
|
|
|
|
|
TLI.getNumRegistersForCallingConv(ReturnType->getContext(), VT);
|
|
|
|
|
MVT PartVT =
|
|
|
|
|
TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), VT);
|
2013-01-11 20:05:37 +00:00
|
|
|
|
|
|
|
|
// 'inreg' on function refers to return value
|
|
|
|
|
ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::InReg))
|
2013-01-11 20:05:37 +00:00
|
|
|
Flags.setInReg();
|
|
|
|
|
|
|
|
|
|
// Propagate extension type if any
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
|
2013-01-11 20:05:37 +00:00
|
|
|
Flags.setSExt();
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298393 91177308-0d34-0410-b5e6-96231b3b80d8
2017-03-21 16:57:19 +00:00
|
|
|
else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt))
|
2013-01-11 20:05:37 +00:00
|
|
|
Flags.setZExt();
|
|
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < NumParts; ++i)
|
2013-10-23 00:44:24 +00:00
|
|
|
Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isFixed=*/true, 0, 0));
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
|
|
|
|
|
/// function arguments in the caller parameter area. This is the actual
|
|
|
|
|
/// alignment, not its logarithm.
|
2015-07-09 02:09:28 +00:00
|
|
|
unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
|
|
|
|
|
const DataLayout &DL) const {
|
|
|
|
|
return DL.getABITypeAlignment(Ty);
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
2015-07-29 18:24:18 +00:00
|
|
|
bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
|
|
|
|
|
const DataLayout &DL, EVT VT,
|
|
|
|
|
unsigned AddrSpace,
|
|
|
|
|
unsigned Alignment,
|
|
|
|
|
bool *Fast) const {
|
|
|
|
|
// Check if the specified alignment is sufficient based on the data layout.
|
|
|
|
|
// TODO: While using the data layout works in practice, a better solution
|
|
|
|
|
// would be to implement this check directly (make this a virtual function).
|
|
|
|
|
// For example, the ABI alignment may change based on software platform while
|
|
|
|
|
// this function should only be affected by hardware implementation.
|
|
|
|
|
Type *Ty = VT.getTypeForEVT(Context);
|
|
|
|
|
if (Alignment >= DL.getABITypeAlignment(Ty)) {
|
|
|
|
|
// Assume that an access that meets the ABI-specified alignment is fast.
|
|
|
|
|
if (Fast != nullptr)
|
|
|
|
|
*Fast = true;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// This is a misaligned access.
|
|
|
|
|
return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Fast);
|
|
|
|
|
}
|
|
|
|
|
|
2016-04-26 17:11:17 +00:00
|
|
|
BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
|
|
|
|
|
return BranchProbability(MinPercentageForPredictableBranch, 100);
|
|
|
|
|
}
|
2015-07-29 18:24:18 +00:00
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// TargetTransformInfo Helpers
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
|
|
|
|
|
enum InstructionOpcodes {
|
|
|
|
|
#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
|
|
|
|
|
#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
|
|
|
|
|
#include "llvm/IR/Instruction.def"
|
|
|
|
|
};
|
|
|
|
|
switch (static_cast<InstructionOpcodes>(Opcode)) {
|
|
|
|
|
case Ret: return 0;
|
|
|
|
|
case Br: return 0;
|
|
|
|
|
case Switch: return 0;
|
|
|
|
|
case IndirectBr: return 0;
|
|
|
|
|
case Invoke: return 0;
|
|
|
|
|
case Resume: return 0;
|
|
|
|
|
case Unreachable: return 0;
|
2015-07-31 17:58:14 +00:00
|
|
|
case CleanupRet: return 0;
|
|
|
|
|
case CatchRet: return 0;
|
[IR] Reformulate LLVM's EH funclet IR
While we have successfully implemented a funclet-oriented EH scheme on
top of LLVM IR, our scheme has some notable deficiencies:
- catchendpad and cleanupendpad are necessary in the current design
but they are difficult to explain to others, even to seasoned LLVM
experts.
- catchendpad and cleanupendpad are optimization barriers. They cannot
be split and force all potentially throwing call-sites to be invokes.
This has a noticable effect on the quality of our code generation.
- catchpad, while similar in some aspects to invoke, is fairly awkward.
It is unsplittable, starts a funclet, and has control flow to other
funclets.
- The nesting relationship between funclets is currently a property of
control flow edges. Because of this, we are forced to carefully
analyze the flow graph to see if there might potentially exist illegal
nesting among funclets. While we have logic to clone funclets when
they are illegally nested, it would be nicer if we had a
representation which forbade them upfront.
Let's clean this up a bit by doing the following:
- Instead, make catchpad more like cleanuppad and landingpad: no control
flow, just a bunch of simple operands; catchpad would be splittable.
- Introduce catchswitch, a control flow instruction designed to model
the constraints of funclet oriented EH.
- Make funclet scoping explicit by having funclet instructions consume
the token produced by the funclet which contains them.
- Remove catchendpad and cleanupendpad. Their presence can be inferred
implicitly using coloring information.
N.B. The state numbering code for the CLR has been updated but the
veracity of it's output cannot be spoken for. An expert should take a
look to make sure the results are reasonable.
Reviewers: rnk, JosephTremoulet, andrew.w.kaylor
Differential Revision: http://reviews.llvm.org/D15139
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@255422 91177308-0d34-0410-b5e6-96231b3b80d8
2015-12-12 05:38:55 +00:00
|
|
|
case CatchPad: return 0;
|
|
|
|
|
case CatchSwitch: return 0;
|
|
|
|
|
case CleanupPad: return 0;
|
2013-01-11 20:05:37 +00:00
|
|
|
case Add: return ISD::ADD;
|
|
|
|
|
case FAdd: return ISD::FADD;
|
|
|
|
|
case Sub: return ISD::SUB;
|
|
|
|
|
case FSub: return ISD::FSUB;
|
|
|
|
|
case Mul: return ISD::MUL;
|
|
|
|
|
case FMul: return ISD::FMUL;
|
|
|
|
|
case UDiv: return ISD::UDIV;
|
2014-04-27 18:47:54 +00:00
|
|
|
case SDiv: return ISD::SDIV;
|
2013-01-11 20:05:37 +00:00
|
|
|
case FDiv: return ISD::FDIV;
|
|
|
|
|
case URem: return ISD::UREM;
|
|
|
|
|
case SRem: return ISD::SREM;
|
|
|
|
|
case FRem: return ISD::FREM;
|
|
|
|
|
case Shl: return ISD::SHL;
|
|
|
|
|
case LShr: return ISD::SRL;
|
|
|
|
|
case AShr: return ISD::SRA;
|
|
|
|
|
case And: return ISD::AND;
|
|
|
|
|
case Or: return ISD::OR;
|
|
|
|
|
case Xor: return ISD::XOR;
|
|
|
|
|
case Alloca: return 0;
|
|
|
|
|
case Load: return ISD::LOAD;
|
|
|
|
|
case Store: return ISD::STORE;
|
|
|
|
|
case GetElementPtr: return 0;
|
|
|
|
|
case Fence: return 0;
|
|
|
|
|
case AtomicCmpXchg: return 0;
|
|
|
|
|
case AtomicRMW: return 0;
|
|
|
|
|
case Trunc: return ISD::TRUNCATE;
|
|
|
|
|
case ZExt: return ISD::ZERO_EXTEND;
|
|
|
|
|
case SExt: return ISD::SIGN_EXTEND;
|
|
|
|
|
case FPToUI: return ISD::FP_TO_UINT;
|
|
|
|
|
case FPToSI: return ISD::FP_TO_SINT;
|
|
|
|
|
case UIToFP: return ISD::UINT_TO_FP;
|
|
|
|
|
case SIToFP: return ISD::SINT_TO_FP;
|
|
|
|
|
case FPTrunc: return ISD::FP_ROUND;
|
|
|
|
|
case FPExt: return ISD::FP_EXTEND;
|
|
|
|
|
case PtrToInt: return ISD::BITCAST;
|
|
|
|
|
case IntToPtr: return ISD::BITCAST;
|
|
|
|
|
case BitCast: return ISD::BITCAST;
|
2013-11-15 01:34:59 +00:00
|
|
|
case AddrSpaceCast: return ISD::ADDRSPACECAST;
|
2013-01-11 20:05:37 +00:00
|
|
|
case ICmp: return ISD::SETCC;
|
|
|
|
|
case FCmp: return ISD::SETCC;
|
|
|
|
|
case PHI: return 0;
|
|
|
|
|
case Call: return 0;
|
|
|
|
|
case Select: return ISD::SELECT;
|
|
|
|
|
case UserOp1: return 0;
|
|
|
|
|
case UserOp2: return 0;
|
|
|
|
|
case VAArg: return 0;
|
|
|
|
|
case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
|
|
|
|
|
case InsertElement: return ISD::INSERT_VECTOR_ELT;
|
|
|
|
|
case ShuffleVector: return ISD::VECTOR_SHUFFLE;
|
|
|
|
|
case ExtractValue: return ISD::MERGE_VALUES;
|
|
|
|
|
case InsertValue: return ISD::MERGE_VALUES;
|
|
|
|
|
case LandingPad: return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
llvm_unreachable("Unknown instruction type encountered!");
|
|
|
|
|
}
|
|
|
|
|
|
2015-08-05 18:08:10 +00:00
|
|
|
std::pair<int, MVT>
|
2015-07-09 02:09:04 +00:00
|
|
|
TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL,
|
|
|
|
|
Type *Ty) const {
|
2013-01-11 20:05:37 +00:00
|
|
|
LLVMContext &C = Ty->getContext();
|
2015-07-09 02:09:04 +00:00
|
|
|
EVT MTy = getValueType(DL, Ty);
|
2013-01-11 20:05:37 +00:00
|
|
|
|
2015-08-05 18:08:10 +00:00
|
|
|
int Cost = 1;
|
2013-01-11 20:05:37 +00:00
|
|
|
// We keep legalizing the type until we find a legal kind. We assume that
|
|
|
|
|
// the only operation that costs anything is the split. After splitting
|
|
|
|
|
// we need to handle two types.
|
|
|
|
|
while (true) {
|
|
|
|
|
LegalizeKind LK = getTypeConversion(C, MTy);
|
|
|
|
|
|
|
|
|
|
if (LK.first == TypeLegal)
|
|
|
|
|
return std::make_pair(Cost, MTy.getSimpleVT());
|
|
|
|
|
|
|
|
|
|
if (LK.first == TypeSplitVector || LK.first == TypeExpandInteger)
|
|
|
|
|
Cost *= 2;
|
|
|
|
|
|
[X86] Part 1 to fix x86-64 fp128 calling convention.
Almost all these changes are conditioned and only apply to the new
x86-64 f128 type configuration, which will be enabled in a follow up
patch. They are required together to make new f128 work. If there is
any error, we should fix or revert them as a whole.
These changes should have no impact to current configurations.
* Relax type legalization checks to accept new f128 type configuration,
whose TypeAction is TypeSoftenFloat, not TypeLegal, but also has
TLI.isTypeLegal true.
* Relax GetSoftenedFloat to return in some cases f128 type SDValue,
which is TLI.isTypeLegal but not "softened" to i128 node.
* Allow customized FABS, FNEG, FCOPYSIGN on new f128 type configuration,
to generate optimized bitwise operators for libm functions.
* Enhance related Lower* functions to handle f128 type.
* Enhance DAGTypeLegalizer::run, SoftenFloatResult, and related functions
to keep new f128 type in register, and convert f128 operators to library calls.
* Fix Combiner, Emitter, Legalizer routines that did not handle f128 type.
* Add ExpandConstant to handle i128 constants, ExpandNode
to handle ISD::Constant node.
* Add one more parameter to getCommonSubClass and firstCommonClass,
to guarantee that returned common sub class will contain the specified
simple value type.
This extra parameter is used by EmitCopyFromReg in InstrEmitter.cpp.
* Fix infinite loop in getTypeLegalizationCost when f128 is the value type.
* Fix printOperand to handle null operand.
* Enhance ISD::BITCAST node to handle f128 constant.
* Expand new f128 type for BR_CC, SELECT_CC, SELECT, SETCC nodes.
* Enhance X86AsmPrinter to emit f128 values in comments.
Differential Revision: http://reviews.llvm.org/D15134
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@254653 91177308-0d34-0410-b5e6-96231b3b80d8
2015-12-03 22:02:40 +00:00
|
|
|
// Do not loop with f128 type.
|
|
|
|
|
if (MTy == LK.second)
|
|
|
|
|
return std::make_pair(Cost, MTy.getSimpleVT());
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
// Keep legalizing the type.
|
|
|
|
|
MTy = LK.second;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-10-14 17:56:00 +00:00
|
|
|
Value *TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilder<> &IRB,
|
|
|
|
|
bool UseTLS) const {
|
|
|
|
|
// compiler-rt provides a variable with a magic name. Targets that do not
|
|
|
|
|
// link with compiler-rt may also provide such a variable.
|
|
|
|
|
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
|
|
|
|
|
const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
|
|
|
|
|
auto UnsafeStackPtr =
|
|
|
|
|
dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
|
|
|
|
|
|
|
|
|
|
Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
|
|
|
|
|
|
|
|
|
|
if (!UnsafeStackPtr) {
|
|
|
|
|
auto TLSModel = UseTLS ?
|
|
|
|
|
GlobalValue::InitialExecTLSModel :
|
|
|
|
|
GlobalValue::NotThreadLocal;
|
|
|
|
|
// The global variable is not defined yet, define it ourselves.
|
|
|
|
|
// We use the initial-exec TLS model because we do not support the
|
|
|
|
|
// variable living anywhere other than in the main executable.
|
|
|
|
|
UnsafeStackPtr = new GlobalVariable(
|
|
|
|
|
*M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
|
|
|
|
|
UnsafeStackPtrVar, nullptr, TLSModel);
|
|
|
|
|
} else {
|
|
|
|
|
// The variable exists, check its type and attributes.
|
|
|
|
|
if (UnsafeStackPtr->getValueType() != StackPtrTy)
|
|
|
|
|
report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
|
|
|
|
|
if (UseTLS != UnsafeStackPtr->isThreadLocal())
|
|
|
|
|
report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
|
|
|
|
|
(UseTLS ? "" : "not ") + "be thread-local");
|
|
|
|
|
}
|
|
|
|
|
return UnsafeStackPtr;
|
|
|
|
|
}
|
|
|
|
|
|
2015-10-26 18:28:25 +00:00
|
|
|
Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
|
|
|
|
|
if (!TM.getTargetTriple().isAndroid())
|
2016-10-14 17:56:00 +00:00
|
|
|
return getDefaultSafeStackPointerLocation(IRB, true);
|
2015-10-26 18:28:25 +00:00
|
|
|
|
|
|
|
|
// Android provides a libc function to retrieve the address of the current
|
|
|
|
|
// thread's unsafe stack pointer.
|
|
|
|
|
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
|
|
|
|
|
Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
|
|
|
|
|
Value *Fn = M->getOrInsertFunction("__safestack_pointer_address",
|
2017-04-11 15:01:18 +00:00
|
|
|
StackPtrTy->getPointerTo(0));
|
2015-10-26 18:28:25 +00:00
|
|
|
return IRB.CreateCall(Fn);
|
|
|
|
|
}
|
|
|
|
|
|
2013-01-11 20:05:37 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Loop Strength Reduction hooks
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
/// isLegalAddressingMode - Return true if the addressing mode represented
|
|
|
|
|
/// by AM is legal for this target, for a load/store of the specified type.
|
2015-07-09 02:09:40 +00:00
|
|
|
bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL,
|
|
|
|
|
const AddrMode &AM, Type *Ty,
|
2015-06-01 05:31:59 +00:00
|
|
|
unsigned AS) const {
|
2013-01-11 20:05:37 +00:00
|
|
|
// The default implementation of this implements a conservative RISCy, r+r and
|
|
|
|
|
// r+i addr mode.
|
|
|
|
|
|
|
|
|
|
// Allows a sign-extended 16-bit immediate field.
|
|
|
|
|
if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
// No global is ever allowed as a base.
|
|
|
|
|
if (AM.BaseGV)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
// Only support r+r,
|
|
|
|
|
switch (AM.Scale) {
|
|
|
|
|
case 0: // "r+i" or just "i", depending on HasBaseReg.
|
|
|
|
|
break;
|
|
|
|
|
case 1:
|
|
|
|
|
if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
|
|
|
|
|
return false;
|
|
|
|
|
// Otherwise we have r+r or r+i.
|
|
|
|
|
break;
|
|
|
|
|
case 2:
|
|
|
|
|
if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
|
|
|
|
|
return false;
|
|
|
|
|
// Allow 2*r as r+r.
|
|
|
|
|
break;
|
2014-02-14 21:10:34 +00:00
|
|
|
default: // Don't allow n * r
|
|
|
|
|
return false;
|
2013-01-11 20:05:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
2016-04-08 21:26:31 +00:00
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Stack Protector
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
// For OpenBSD return its special guard variable. Otherwise return nullptr,
|
|
|
|
|
// so that SelectionDAG handle SSP.
|
|
|
|
|
Value *TargetLoweringBase::getIRStackGuard(IRBuilder<> &IRB) const {
|
|
|
|
|
if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
|
|
|
|
|
Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
|
|
|
|
|
PointerType *PtrTy = Type::getInt8PtrTy(M.getContext());
|
2016-08-22 18:26:27 +00:00
|
|
|
return M.getOrInsertGlobal("__guard_local", PtrTy);
|
2016-04-08 21:26:31 +00:00
|
|
|
}
|
|
|
|
|
return nullptr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Currently only support "standard" __stack_chk_guard.
|
|
|
|
|
// TODO: add LOAD_STACK_GUARD support.
|
|
|
|
|
void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
|
|
|
|
|
M.getOrInsertGlobal("__stack_chk_guard", Type::getInt8PtrTy(M.getContext()));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Currently only support "standard" __stack_chk_guard.
|
|
|
|
|
// TODO: add LOAD_STACK_GUARD support.
|
2016-04-19 20:14:52 +00:00
|
|
|
Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {
|
2016-06-09 14:23:38 +00:00
|
|
|
return M.getGlobalVariable("__stack_chk_guard", true);
|
2016-04-08 21:26:31 +00:00
|
|
|
}
|
[stack-protection] Add support for MSVC buffer security check
Summary:
This patch is adding support for the MSVC buffer security check implementation
The buffer security check is turned on with the '/GS' compiler switch.
* https://msdn.microsoft.com/en-us/library/8dbf701c.aspx
* To be added to clang here: http://reviews.llvm.org/D20347
Some overview of buffer security check feature and implementation:
* https://msdn.microsoft.com/en-us/library/aa290051(VS.71).aspx
* http://www.ksyash.com/2011/01/buffer-overflow-protection-3/
* http://blog.osom.info/2012/02/understanding-vs-c-compilers-buffer.html
For the following example:
```
int example(int offset, int index) {
char buffer[10];
memset(buffer, 0xCC, index);
return buffer[index];
}
```
The MSVC compiler is adding these instructions to perform stack integrity check:
```
push ebp
mov ebp,esp
sub esp,50h
[1] mov eax,dword ptr [__security_cookie (01068024h)]
[2] xor eax,ebp
[3] mov dword ptr [ebp-4],eax
push ebx
push esi
push edi
mov eax,dword ptr [index]
push eax
push 0CCh
lea ecx,[buffer]
push ecx
call _memset (010610B9h)
add esp,0Ch
mov eax,dword ptr [index]
movsx eax,byte ptr buffer[eax]
pop edi
pop esi
pop ebx
[4] mov ecx,dword ptr [ebp-4]
[5] xor ecx,ebp
[6] call @__security_check_cookie@4 (01061276h)
mov esp,ebp
pop ebp
ret
```
The instrumentation above is:
* [1] is loading the global security canary,
* [3] is storing the local computed ([2]) canary to the guard slot,
* [4] is loading the guard slot and ([5]) re-compute the global canary,
* [6] is validating the resulting canary with the '__security_check_cookie' and performs error handling.
Overview of the current stack-protection implementation:
* lib/CodeGen/StackProtector.cpp
* There is a default stack-protection implementation applied on intermediate representation.
* The target can overload 'getIRStackGuard' method if it has a standard location for the stack protector cookie.
* An intrinsic 'Intrinsic::stackprotector' is added to the prologue. It will be expanded by the instruction selection pass (DAG or Fast).
* Basic Blocks are added to every instrumented function to receive the code for handling stack guard validation and errors handling.
* Guard manipulation and comparison are added directly to the intermediate representation.
* lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
* lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
* There is an implementation that adds instrumentation during instruction selection (for better handling of sibbling calls).
* see long comment above 'class StackProtectorDescriptor' declaration.
* The target needs to override 'getSDagStackGuard' to activate SDAG stack protection generation. (note: getIRStackGuard MUST be nullptr).
* 'getSDagStackGuard' returns the appropriate stack guard (security cookie)
* The code is generated by 'SelectionDAGBuilder.cpp' and 'SelectionDAGISel.cpp'.
* include/llvm/Target/TargetLowering.h
* Contains function to retrieve the default Guard 'Value'; should be overriden by each target to select which implementation is used and provide Guard 'Value'.
* lib/Target/X86/X86ISelLowering.cpp
* Contains the x86 specialisation; Guard 'Value' used by the SelectionDAG algorithm.
Function-based Instrumentation:
* The MSVC doesn't inline the stack guard comparison in every function. Instead, a call to '__security_check_cookie' is added to the epilogue before every return instructions.
* To support function-based instrumentation, this patch is
* adding a function to get the function-based check (llvm 'Value', see include/llvm/Target/TargetLowering.h),
* If provided, the stack protection instrumentation won't be inlined and a call to that function will be added to the prologue.
* modifying (SelectionDAGISel.cpp) do avoid producing basic blocks used for inline instrumentation,
* generating the function-based instrumentation during the ISEL pass (SelectionDAGBuilder.cpp),
* if FastISEL (not SelectionDAG), using the fallback which rely on the same function-based implemented over intermediate representation (StackProtector.cpp).
Modifications
* adding support for MSVC (lib/Target/X86/X86ISelLowering.cpp)
* adding support function-based instrumentation (lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp, .h)
Results
* IR generated instrumentation:
```
clang-cl /GS test.cc /Od /c -mllvm -print-isel-input
```
```
*** Final LLVM Code input to ISel ***
; Function Attrs: nounwind sspstrong
define i32 @"\01?example@@YAHHH@Z"(i32 %offset, i32 %index) #0 {
entry:
%StackGuardSlot = alloca i8* <<<-- Allocated guard slot
%0 = call i8* @llvm.stackguard() <<<-- Loading Stack Guard value
call void @llvm.stackprotector(i8* %0, i8** %StackGuardSlot) <<<-- Prologue intrinsic call (store to Guard slot)
%index.addr = alloca i32, align 4
%offset.addr = alloca i32, align 4
%buffer = alloca [10 x i8], align 1
store i32 %index, i32* %index.addr, align 4
store i32 %offset, i32* %offset.addr, align 4
%arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %buffer, i32 0, i32 0
%1 = load i32, i32* %index.addr, align 4
call void @llvm.memset.p0i8.i32(i8* %arraydecay, i8 -52, i32 %1, i32 1, i1 false)
%2 = load i32, i32* %index.addr, align 4
%arrayidx = getelementptr inbounds [10 x i8], [10 x i8]* %buffer, i32 0, i32 %2
%3 = load i8, i8* %arrayidx, align 1
%conv = sext i8 %3 to i32
%4 = load volatile i8*, i8** %StackGuardSlot <<<-- Loading Guard slot
call void @__security_check_cookie(i8* %4) <<<-- Epilogue function-based check
ret i32 %conv
}
```
* SelectionDAG generated instrumentation:
```
clang-cl /GS test.cc /O1 /c /FA
```
```
"?example@@YAHHH@Z": # @"\01?example@@YAHHH@Z"
# BB#0: # %entry
pushl %esi
subl $16, %esp
movl ___security_cookie, %eax <<<-- Loading Stack Guard value
movl 28(%esp), %esi
movl %eax, 12(%esp) <<<-- Store to Guard slot
leal 2(%esp), %eax
pushl %esi
pushl $204
pushl %eax
calll _memset
addl $12, %esp
movsbl 2(%esp,%esi), %esi
movl 12(%esp), %ecx <<<-- Loading Guard slot
calll @__security_check_cookie@4 <<<-- Epilogue function-based check
movl %esi, %eax
addl $16, %esp
popl %esi
retl
```
Reviewers: kcc, pcc, eugenis, rnk
Subscribers: majnemer, llvm-commits, hans, thakis, rnk
Differential Revision: http://reviews.llvm.org/D20346
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@272053 91177308-0d34-0410-b5e6-96231b3b80d8
2016-06-07 20:15:35 +00:00
|
|
|
|
|
|
|
|
Value *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
|
|
|
|
|
return nullptr;
|
|
|
|
|
}
|
2016-09-26 15:32:33 +00:00
|
|
|
|
2016-10-25 19:53:51 +00:00
|
|
|
unsigned TargetLoweringBase::getMinimumJumpTableEntries() const {
|
|
|
|
|
return MinimumJumpTableEntries;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) {
|
|
|
|
|
MinimumJumpTableEntries = Val;
|
|
|
|
|
}
|
|
|
|
|
|
[InlineCost] Improve the cost heuristic for Switch
Summary:
The motivation example is like below which has 13 cases but only 2 distinct targets
```
lor.lhs.false2: ; preds = %if.then
switch i32 %Status, label %if.then27 [
i32 -7012, label %if.end35
i32 -10008, label %if.end35
i32 -10016, label %if.end35
i32 15000, label %if.end35
i32 14013, label %if.end35
i32 10114, label %if.end35
i32 10107, label %if.end35
i32 10105, label %if.end35
i32 10013, label %if.end35
i32 10011, label %if.end35
i32 7008, label %if.end35
i32 7007, label %if.end35
i32 5002, label %if.end35
]
```
which is compiled into a balanced binary tree like this on AArch64 (similar on X86)
```
.LBB853_9: // %lor.lhs.false2
mov w8, #10012
cmp w19, w8
b.gt .LBB853_14
// BB#10: // %lor.lhs.false2
mov w8, #5001
cmp w19, w8
b.gt .LBB853_18
// BB#11: // %lor.lhs.false2
mov w8, #-10016
cmp w19, w8
b.eq .LBB853_23
// BB#12: // %lor.lhs.false2
mov w8, #-10008
cmp w19, w8
b.eq .LBB853_23
// BB#13: // %lor.lhs.false2
mov w8, #-7012
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_14: // %lor.lhs.false2
mov w8, #14012
cmp w19, w8
b.gt .LBB853_21
// BB#15: // %lor.lhs.false2
mov w8, #-10105
add w8, w19, w8
cmp w8, #9 // =9
b.hi .LBB853_17
// BB#16: // %lor.lhs.false2
orr w9, wzr, #0x1
lsl w8, w9, w8
mov w9, #517
and w8, w8, w9
cbnz w8, .LBB853_23
.LBB853_17: // %lor.lhs.false2
mov w8, #10013
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_18: // %lor.lhs.false2
mov w8, #-7007
add w8, w19, w8
cmp w8, #2 // =2
b.lo .LBB853_23
// BB#19: // %lor.lhs.false2
mov w8, #5002
cmp w19, w8
b.eq .LBB853_23
// BB#20: // %lor.lhs.false2
mov w8, #10011
cmp w19, w8
b.eq .LBB853_23
b .LBB853_3
.LBB853_21: // %lor.lhs.false2
mov w8, #14013
cmp w19, w8
b.eq .LBB853_23
// BB#22: // %lor.lhs.false2
mov w8, #15000
cmp w19, w8
b.ne .LBB853_3
```
However, the inline cost model estimates the cost to be linear with the number
of distinct targets and the cost of the above switch is just 2 InstrCosts.
The function containing this switch is then inlined about 900 times.
This change use the general way of switch lowering for the inline heuristic. It
etimate the number of case clusters with the suitability check for a jump table
or bit test. Considering the binary search tree built for the clusters, this
change modifies the model to be linear with the size of the balanced binary
tree. The model is off by default for now :
-inline-generic-switch-cost=false
This change was originally proposed by Haicheng in D29870.
Reviewers: hans, bmakam, chandlerc, eraman, haicheng, mcrosier
Reviewed By: hans
Subscribers: joerg, aemerson, llvm-commits, rengolin
Differential Revision: https://reviews.llvm.org/D31085
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@301649 91177308-0d34-0410-b5e6-96231b3b80d8
2017-04-28 16:04:03 +00:00
|
|
|
unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
|
|
|
|
|
return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-26 15:32:33 +00:00
|
|
|
unsigned TargetLoweringBase::getMaximumJumpTableSize() const {
|
|
|
|
|
return MaximumJumpTableSize;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {
|
|
|
|
|
MaximumJumpTableSize = Val;
|
|
|
|
|
}
|
2016-10-20 16:55:45 +00:00
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Reciprocal Estimates
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
/// Get the reciprocal estimate attribute string for a function that will
|
|
|
|
|
/// override the target defaults.
|
|
|
|
|
static StringRef getRecipEstimateForFunc(MachineFunction &MF) {
|
|
|
|
|
const Function *F = MF.getFunction();
|
2017-01-13 22:24:25 +00:00
|
|
|
return F->getFnAttribute("reciprocal-estimates").getValueAsString();
|
2016-10-20 16:55:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Construct a string for the given reciprocal operation of the given type.
|
|
|
|
|
/// This string should match the corresponding option to the front-end's
|
|
|
|
|
/// "-mrecip" flag assuming those strings have been passed through in an
|
|
|
|
|
/// attribute string. For example, "vec-divf" for a division of a vXf32.
|
|
|
|
|
static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
|
|
|
|
|
std::string Name = VT.isVector() ? "vec-" : "";
|
|
|
|
|
|
|
|
|
|
Name += IsSqrt ? "sqrt" : "div";
|
|
|
|
|
|
|
|
|
|
// TODO: Handle "half" or other float types?
|
|
|
|
|
if (VT.getScalarType() == MVT::f64) {
|
|
|
|
|
Name += "d";
|
|
|
|
|
} else {
|
|
|
|
|
assert(VT.getScalarType() == MVT::f32 &&
|
|
|
|
|
"Unexpected FP type for reciprocal estimate");
|
|
|
|
|
Name += "f";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return Name;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Return the character position and value (a single numeric character) of a
|
|
|
|
|
/// customized refinement operation in the input string if it exists. Return
|
|
|
|
|
/// false if there is no customized refinement step count.
|
|
|
|
|
static bool parseRefinementStep(StringRef In, size_t &Position,
|
|
|
|
|
uint8_t &Value) {
|
|
|
|
|
const char RefStepToken = ':';
|
|
|
|
|
Position = In.find(RefStepToken);
|
|
|
|
|
if (Position == StringRef::npos)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
StringRef RefStepString = In.substr(Position + 1);
|
|
|
|
|
// Allow exactly one numeric character for the additional refinement
|
|
|
|
|
// step parameter.
|
|
|
|
|
if (RefStepString.size() == 1) {
|
|
|
|
|
char RefStepChar = RefStepString[0];
|
|
|
|
|
if (RefStepChar >= '0' && RefStepChar <= '9') {
|
|
|
|
|
Value = RefStepChar - '0';
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
report_fatal_error("Invalid refinement step for -recip.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// For the input attribute string, return one of the ReciprocalEstimate enum
|
|
|
|
|
/// status values (enabled, disabled, or not specified) for this operation on
|
|
|
|
|
/// the specified data type.
|
|
|
|
|
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
|
|
|
|
|
if (Override.empty())
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
|
|
|
|
|
SmallVector<StringRef, 4> OverrideVector;
|
|
|
|
|
SplitString(Override, OverrideVector, ",");
|
|
|
|
|
unsigned NumArgs = OverrideVector.size();
|
|
|
|
|
|
|
|
|
|
// Check if "all", "none", or "default" was specified.
|
|
|
|
|
if (NumArgs == 1) {
|
|
|
|
|
// Look for an optional setting of the number of refinement steps needed
|
|
|
|
|
// for this type of reciprocal operation.
|
|
|
|
|
size_t RefPos;
|
|
|
|
|
uint8_t RefSteps;
|
|
|
|
|
if (parseRefinementStep(Override, RefPos, RefSteps)) {
|
|
|
|
|
// Split the string for further processing.
|
|
|
|
|
Override = Override.substr(0, RefPos);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// All reciprocal types are enabled.
|
|
|
|
|
if (Override == "all")
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Enabled;
|
|
|
|
|
|
|
|
|
|
// All reciprocal types are disabled.
|
|
|
|
|
if (Override == "none")
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Disabled;
|
|
|
|
|
|
|
|
|
|
// Target defaults for enablement are used.
|
|
|
|
|
if (Override == "default")
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The attribute string may omit the size suffix ('f'/'d').
|
|
|
|
|
std::string VTName = getReciprocalOpName(IsSqrt, VT);
|
|
|
|
|
std::string VTNameNoSize = VTName;
|
2016-10-21 14:58:30 +00:00
|
|
|
VTNameNoSize.pop_back();
|
2016-10-20 16:55:45 +00:00
|
|
|
static const char DisabledPrefix = '!';
|
|
|
|
|
|
|
|
|
|
for (StringRef RecipType : OverrideVector) {
|
|
|
|
|
size_t RefPos;
|
|
|
|
|
uint8_t RefSteps;
|
|
|
|
|
if (parseRefinementStep(RecipType, RefPos, RefSteps))
|
|
|
|
|
RecipType = RecipType.substr(0, RefPos);
|
|
|
|
|
|
|
|
|
|
// Ignore the disablement token for string matching.
|
|
|
|
|
bool IsDisabled = RecipType[0] == DisabledPrefix;
|
|
|
|
|
if (IsDisabled)
|
|
|
|
|
RecipType = RecipType.substr(1);
|
|
|
|
|
|
|
|
|
|
if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
|
|
|
|
|
return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled
|
|
|
|
|
: TargetLoweringBase::ReciprocalEstimate::Enabled;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// For the input attribute string, return the customized refinement step count
|
|
|
|
|
/// for this operation on the specified data type. If the step count does not
|
|
|
|
|
/// exist, return the ReciprocalEstimate enum value for unspecified.
|
|
|
|
|
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
|
|
|
|
|
if (Override.empty())
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
|
|
|
|
|
SmallVector<StringRef, 4> OverrideVector;
|
|
|
|
|
SplitString(Override, OverrideVector, ",");
|
|
|
|
|
unsigned NumArgs = OverrideVector.size();
|
|
|
|
|
|
|
|
|
|
// Check if "all", "default", or "none" was specified.
|
|
|
|
|
if (NumArgs == 1) {
|
|
|
|
|
// Look for an optional setting of the number of refinement steps needed
|
|
|
|
|
// for this type of reciprocal operation.
|
|
|
|
|
size_t RefPos;
|
|
|
|
|
uint8_t RefSteps;
|
|
|
|
|
if (!parseRefinementStep(Override, RefPos, RefSteps))
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
|
|
|
|
|
// Split the string for further processing.
|
|
|
|
|
Override = Override.substr(0, RefPos);
|
|
|
|
|
assert(Override != "none" &&
|
|
|
|
|
"Disabled reciprocals, but specifed refinement steps?");
|
|
|
|
|
|
|
|
|
|
// If this is a general override, return the specified number of steps.
|
|
|
|
|
if (Override == "all" || Override == "default")
|
|
|
|
|
return RefSteps;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The attribute string may omit the size suffix ('f'/'d').
|
|
|
|
|
std::string VTName = getReciprocalOpName(IsSqrt, VT);
|
|
|
|
|
std::string VTNameNoSize = VTName;
|
2016-10-21 14:58:30 +00:00
|
|
|
VTNameNoSize.pop_back();
|
2016-10-20 16:55:45 +00:00
|
|
|
|
|
|
|
|
for (StringRef RecipType : OverrideVector) {
|
|
|
|
|
size_t RefPos;
|
|
|
|
|
uint8_t RefSteps;
|
|
|
|
|
if (!parseRefinementStep(RecipType, RefPos, RefSteps))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
RecipType = RecipType.substr(0, RefPos);
|
|
|
|
|
if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
|
|
|
|
|
return RefSteps;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return TargetLoweringBase::ReciprocalEstimate::Unspecified;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT,
|
|
|
|
|
MachineFunction &MF) const {
|
|
|
|
|
return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT,
|
|
|
|
|
MachineFunction &MF) const {
|
|
|
|
|
return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,
|
|
|
|
|
MachineFunction &MF) const {
|
|
|
|
|
return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int TargetLoweringBase::getDivRefinementSteps(EVT VT,
|
|
|
|
|
MachineFunction &MF) const {
|
|
|
|
|
return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
|
|
|
|
|
}
|
2017-04-28 20:25:05 +00:00
|
|
|
|
|
|
|
|
void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const {
|
|
|
|
|
MF.getRegInfo().freezeReservedRegs(MF);
|
|
|
|
|
}
|