Match MSVC's handling of commas during macro argument expansion

This allows clang to parse the type_traits header in Visual Studio 2012,
which is included widely in practice.

This is a rework of r163022 by João Matos.  The original patch broke
preprocessing of gtest headers, which this patch addresses.

Patch by Will Wilson!

llvm-svn: 184968

clang/include/clang/Lex/Token.h

@@ -77,7 +77,8 @@ public:
     NeedsCleaning = 0x08,  // Contained an escaped newline or trigraph.
     LeadingEmptyMacro = 0x10, // Empty macro exists before this token.
     HasUDSuffix = 0x20,    // This string or character literal has a ud-suffix.
-    HasUCN = 0x40          // This identifier contains a UCN.
+    HasUCN = 0x40,         // This identifier contains a UCN.
+    IgnoredComma = 0x80,   // This comma is not a macro argument separator (MS).
   };
 
   tok::TokenKind getKind() const { return (tok::TokenKind)Kind; }

clang/lib/Lex/PPMacroExpansion.cpp

@@ -458,7 +458,12 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
         }
       } else if (Tok.is(tok::l_paren)) {
         ++NumParens;
-      } else if (Tok.is(tok::comma) && NumParens == 0) {
+      } else if (Tok.is(tok::comma) && NumParens == 0 &&
+                 !(Tok.getFlags() & Token::IgnoredComma)) {
+        // In Microsoft-compatibility mode, single commas from nested macro
+        // expansions should not be considered as argument separators. We test
+        // for this with the IgnoredComma token flag above.
+
         // Comma ends this argument if there are more fixed arguments expected.
         // However, if this is a variadic macro, and this is part of the
         // variadic part, then the comma is just an argument token.

clang/lib/Lex/TokenLexer.cpp

@@ -278,6 +278,14 @@ void TokenLexer::ExpandFunctionArguments() {
         unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
         ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
 
+        // In Microsoft-compatibility mode, we follow MSVC's preprocessing
+        // behavior by not considering single commas from nested macro
+        // expansions as argument separators. Set a flag on the token so we can
+        // test for this later when the macro expansion is processed.
+        if (PP.getLangOpts().MicrosoftMode && NumToks == 1 &&
+            ResultToks.back().is(tok::comma))
+          ResultToks.back().setFlag(Token::IgnoredComma);
+
         // If the '##' came from expanding an argument, turn it into 'unknown'
         // to avoid pasting.
         for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {

clang/test/Preprocessor/microsoft-ext.c

@@ -22,3 +22,15 @@ class GMOCK_ACTION_CLASS_(name, value_params) {\
 ACTION_TEMPLATE(InvokeArgument,
                 HAS_1_TEMPLATE_PARAMS(int, k),
                 AND_2_VALUE_PARAMS(p0, p1));
+
+// This tests compatibility with behaviour needed for type_traits in VS2012
+// Test based on _VARIADIC_EXPAND_0X macros in xstddef of VS2012
+#define _COMMA ,
+
+#define MAKER(_arg1, _comma, _arg2)                                            \
+  void func(_arg1 _comma _arg2) {}
+#define MAKE_FUNC(_makerP1, _makerP2, _arg1, _comma, _arg2)                    \
+  _makerP1##_makerP2(_arg1, _comma, _arg2)
+
+MAKE_FUNC(MAK, ER, int a, _COMMA, int b);
+// CHECK: void func(int a , int b) {}