Fix computation of linkage within nested lambdas.

When nested C++11 lambdas are used in NSDMI's - this patch prevents infinite recursion by computing the linkage of any nested lambda by determining the linkage of the outermost enclosing lambda (which might inherit its linkage from its parent).

See http://llvm-reviews.chandlerc.com/D1783 for Doug's approval.

[On a related note, I need this patch so as to pass tests of transformations of nested lambdas returned from member functions]

llvm-svn: 191727

clang/lib/AST/Decl.cpp

@@ -1092,6 +1092,19 @@ static LinkageInfo getLVForLocalDecl(const NamedDecl *D,
                      LV.isVisibilityExplicit());
 }
 
+static inline const CXXRecordDecl*
+getOutermostEnclosingLambda(const CXXRecordDecl *Record) {
+  const CXXRecordDecl *Ret = Record;
+  while (Record && Record->isLambda()) {
+    Ret = Record;
+    if (!Record->getParent()) break;
+    // Get the Containing Class of this Lambda Class
+    Record = dyn_cast_or_null<CXXRecordDecl>(
+      Record->getParent()->getParent());
+  }
+  return Ret;
+}
+
 static LinkageInfo computeLVForDecl(const NamedDecl *D,
                                     LVComputationKind computation) {
   // Objective-C: treat all Objective-C declarations as having external
@@ -1122,9 +1135,24 @@ static LinkageInfo computeLVForDecl(const NamedDecl *D,
           return LinkageInfo::internal();
         }
 
-        // This lambda has its linkage/visibility determined by its owner.
-        return getLVForClosure(D->getDeclContext()->getRedeclContext(),
-                               Record->getLambdaContextDecl(), computation);
+        // This lambda has its linkage/visibility determined:
+        //  - either by the outermost lambda if that lambda has no mangling 
+        //    number. 
+        //  - or by the parent of the outer most lambda
+        // This prevents infinite recursion in settings such as nested lambdas 
+        // used in NSDMI's, for e.g. 
+        //  struct L {
+        //    int t{};
+        //    int t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+        //  };
+        const CXXRecordDecl *OuterMostLambda = 
+            getOutermostEnclosingLambda(Record);
+        if (!OuterMostLambda->getLambdaManglingNumber())
+          return LinkageInfo::internal();
+        
+        return getLVForClosure(
+                  OuterMostLambda->getDeclContext()->getRedeclContext(),
+                  OuterMostLambda->getLambdaContextDecl(), computation);
       }
       
       break;

clang/test/CodeGenCXX/lambda-expressions-nested-linkage.cpp

@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin10.0.0 -fblocks -emit-llvm -o - %s -fexceptions -std=c++11 | FileCheck %s
+
+// CHECK-LABEL: define void @_ZN19non_inline_function3fooEv()
+// CHECK-LABEL: define internal void @"_ZZN19non_inline_function3fooEvENK3$_0clEi"(%class.anon
+// CHECK-LABEL: define internal signext i8 @"_ZZZN19non_inline_function3fooEvENK3$_0clEiENKUlcE_clEc"(%class.anon
+namespace non_inline_function {
+void foo() {
+  auto L = [](int a) {
+    return [](char b) {
+     return b;
+    };
+  };
+  L(3)('a');
+}
+}
+
+namespace non_template {
+  struct L {
+    int t = ([](int a) { return [](int b) { return b; };})(2)(3);    
+  };
+  L l; 
+}
+
+namespace lambdas_in_NSDMIs_template_class {
+template<class T>
+struct L {
+    T t2 = ([](int a) { return [](int b) { return b; };})(T{})(T{});    
+};
+L<int> l;
+}
+
+// CHECK-LABEL: define linkonce_odr i32 @_ZN15inline_function3fooEv
+// CHECK: define linkonce_odr void @_ZZN15inline_function3fooEvENKUliE_clEi
+// CHECK: define linkonce_odr signext i8 @_ZZZN15inline_function3fooEvENKUliE_clEiENKUlcE_clEc
+namespace inline_function {
+inline int foo() {
+  auto L = [](int a) {
+    return [](char b) {
+     return b;
+    };
+  };
+  L(3)('a');
+}
+int use = foo();
+}
+// CHECK: define linkonce_odr void @_ZNK32lambdas_in_NSDMIs_template_class1LIiEUliE_clEi(%class.anon
+// CHECK: define linkonce_odr i32 @_ZZNK32lambdas_in_NSDMIs_template_class1LIiEUliE_clEiENKUliE_clEi(%class.anon
+
+// CHECK: define linkonce_odr void @_ZNK12non_template1L1tMUliE_clEi(%class.anon
+// CHECK: define linkonce_odr i32 @_ZZNK12non_template1L1tMUliE_clEiENKUliE_clEi(%class.anon

clang/test/SemaCXX/lambda-expressions.cpp

@@ -265,3 +265,21 @@ namespace TypeDeduction {
 #endif
   }
 }
+
+
+namespace lambdas_in_NSDMIs {
+  template<class T>
+  struct L {
+      T t{};
+      T t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+  };
+  L<int> l; 
+  
+  namespace non_template {
+    struct L {
+      int t = 0;
+      int t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+    };
+    L l; 
+  }
+}