[InlineCost] Mark functions accessing varargs as not viable.

This prevents functions accessing varargs from being inlined if they
have the alwaysinline attribute.

Reviewers: efriedma, rnk, davide

Reviewed By: efriedma

Differential Revision: https://reviews.llvm.org/D42556


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@323619 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Analysis/InlineCost.cpp

@@ -2040,12 +2040,18 @@ bool llvm::isInlineViable(Function &F) {
           cast<CallInst>(CS.getInstruction())->canReturnTwice())
         return false;
 
-      // Disallow inlining functions that call @llvm.localescape. Doing this
-      // correctly would require major changes to the inliner.
-      if (CS.getCalledFunction() &&
-          CS.getCalledFunction()->getIntrinsicID() ==
-              llvm::Intrinsic::localescape)
-        return false;
+      if (CS.getCalledFunction())
+        switch (CS.getCalledFunction()->getIntrinsicID()) {
+        default:
+          break;
+        // Disallow inlining functions that call @llvm.localescape. Doing this
+        // correctly would require major changes to the inliner.
+        case llvm::Intrinsic::localescape:
+        // Disallow inlining of functions that access VarArgs.
+        case llvm::Intrinsic::vastart:
+        case llvm::Intrinsic::vaend:
+          return false;
+        }
     }
   }
 

test/Transforms/Inline/inline-varargs.ll

@@ -64,12 +64,25 @@ entry:
   ret i32 %va1
 }
 
+define internal i32 @varg_accessed_alwaysinline(...) alwaysinline {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs.ptr = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs.ptr)
+  %va1 = va_arg i8** %vargs, i32
+  call void @llvm.va_end(i8* %vargs.ptr)
+  ret i32 %va1
+}
+
 define i32 @call_vargs() {
-  %res = call i32 (...) @varg_accessed(i32 10)
+  %res1 = call i32 (...) @varg_accessed(i32 10)
+  %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
+  %res = add i32 %res1, %res2
   ret i32 %res
 }
 ; CHECK-LABEL: @call_vargs
-; CHECK: %res = call i32 (...) @varg_accessed(i32 10)
+; CHECK: %res1 = call i32 (...) @varg_accessed(i32 10)
+; CHECK-NEXT: %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
 
 declare void @llvm.va_start(i8*)
 declare void @llvm.va_end(i8*)