Take an entirely different approach to handling the "parsing" of

__import__ within the preprocessor, since the prior one foolishly
assumed that Preprocessor::Lex() was re-entrant. We now handle
__import__ at the top level (only), after macro expansion. This should
fix the buildbot failures.

llvm-svn: 138704

clang/include/clang/Basic/IdentifierTable.h

@@ -252,7 +252,7 @@ private:
   void RecomputeNeedsHandleIdentifier() {
     NeedsHandleIdentifier =
       (isPoisoned() | hasMacroDefinition() | isCPlusPlusOperatorKeyword() |
-       isExtensionToken() | (getTokenID() == tok::kw___import__));
+       isExtensionToken());
   }
 };
 

clang/include/clang/Lex/Preprocessor.h

@@ -118,6 +118,9 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// \brief Whether we have already loaded macros from the external source.
   mutable bool ReadMacrosFromExternalSource : 1;
 
+  /// \brief Tracks the depth of Lex() Calls.
+  unsigned LexDepth;
+  
   /// Identifiers - This is mapping/lookup information for all identifiers in
   /// the program, including program keywords.
   mutable IdentifierTable Identifiers;
@@ -531,6 +534,7 @@ public:
   /// Lex - To lex a token from the preprocessor, just pull a token from the
   /// current lexer or macro object.
   void Lex(Token &Result) {
+    ++LexDepth;
     if (CurLexer)
       CurLexer->Lex(Result);
     else if (CurPTHLexer)
@@ -539,6 +543,11 @@ public:
       CurTokenLexer->Lex(Result);
     else
       CachingLex(Result);
+    --LexDepth;
+    
+    // If we have the __import__ keyword, handle the module import now.
+    if (Result.getKind() == tok::kw___import__ && LexDepth == 0)
+      HandleModuleImport(Result);
   }
 
   /// LexNonComment - Lex a token.  If it's a comment, keep lexing until we get

clang/lib/Lex/Preprocessor.cpp

@@ -88,6 +88,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
   // We haven't read anything from the external source.
   ReadMacrosFromExternalSource = false;
 
+  LexDepth = 0;
+      
   // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
   // This gets unpoisoned where it is allowed.
   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
@@ -484,7 +486,7 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
     if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) {
       if (MI->isEnabled()) {
         if (!HandleMacroExpandedIdentifier(Identifier, MI))
-          goto finish;
+          return;
       } else {
         // C99 6.10.3.4p2 says that a disabled macro may never again be
         // expanded, even if it's in a context where it could be expanded in the
@@ -506,12 +508,6 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
   // like "#define TY typeof", "TY(1) x".
   if (II.isExtensionToken() && !DisableMacroExpansion)
     Diag(Identifier, diag::ext_token_used);
-  
-finish:
-  // If we have the start of a module import, handle it now.
-  if (Identifier.is(tok::kw___import__) &&
-      !InMacroArgs && !DisableMacroExpansion)
-    HandleModuleImport(Identifier);
 }
 
 void Preprocessor::HandleModuleImport(Token &Import) {

clang/test/Modules/lookup.cpp

@@ -18,7 +18,6 @@ void test(int i, float f) {
 // RUN: %clang_cc1 -emit-module -x c++ -o %T/lookup_right_cxx.pcm %S/Inputs/lookup_right.hpp
 // RUN: %clang_cc1 -x c++ -I %T %s -verify
 // RUN: %clang_cc1 -ast-print -x c++ -I %T %s | FileCheck -check-prefix=CHECK-PRINT %s
-// XFAIL: win32
 
 // CHECK-PRINT: int *f0(int *);
 // CHECK-PRINT: float *f0(float *);