revert r133003 and fix the bug properly: the issue was that ## in a token

lexer is not a paste operator, it is a normal token.  This fixes a conformance
issue shown here:
http://p99.gforge.inria.fr/c99-conformance/c99-conformance-clang-2.9.html

and it defines away the crash from before.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@133005 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Lex/TokenLexer.cpp

@@ -327,7 +327,8 @@ void TokenLexer::Lex(Token &Tok) {
   bool TokenIsFromPaste = false;
 
   // If this token is followed by a token paste (##) operator, paste the tokens!
-  if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)) {
+  // Note that ## is a normal token when not expanding a macro.
+  if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) {
     // When handling the microsoft /##/ extension, the final token is
     // returned by PasteTokens, not the pasted token.
     if (PasteTokens(Tok))
@@ -487,10 +488,9 @@ bool TokenLexer::PasteTokens(Token &Tok) {
           // Explicitly convert the token location to have proper instantiation
           // information so that the user knows where it came from.
           SourceManager &SM = PP.getSourceManager();
-          SourceLocation Loc = PasteOpLoc;
-          if (InstantiateLocStart.isValid())
-            Loc = SM.createInstantiationLoc(Loc, InstantiateLocStart,
-                                            InstantiateLocEnd, 2);
+          SourceLocation Loc =
+            SM.createInstantiationLoc(PasteOpLoc, InstantiateLocStart,
+                                      InstantiateLocEnd, 2);
           // If we're in microsoft extensions mode, downgrade this from a hard
           // error to a warning that defaults to an error.  This allows
           // disabling it.

test/Preprocessor/macro_paste_bad.c

@@ -32,10 +32,3 @@ XX     // expected-error {{attempt to use a poisoned identifier}}
 #define VA __VA_ ## ARGS__
 int VA;   // expected-warning {{__VA_ARGS__ can only appear in the expansion of a C99 variadic macro}}
 
-
-// PR9981
-#define M1(A) A
-#define M2(X) 
-M1(M2(##))   // expected-error {{pasting formed '()', an invalid preprocessing token}}
-
-

test/Preprocessor/macro_paste_simple.c

@@ -1,5 +1,14 @@
-// RUN: %clang_cc1 %s -E | grep "barbaz123"
+// RUN: %clang_cc1 %s -E | FileCheck %s
 
 #define FOO bar ## baz ## 123
 
-FOO
+// CHECK: A: barbaz123
+A: FOO
+
+// PR9981
+#define M1(A) A
+#define M2(X) X
+B: M1(M2(##))
+
+// CHECK: B: ##
+