Add -frewrite-imports flag.

If specified, when preprocessing, the contents of imported .pcm files will be
included in preprocessed output. The resulting preprocessed file can then be
compiled standalone without the module sources or .pcm files.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@305116 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/Driver/Options.td

@@ -932,6 +932,10 @@ def frewrite_includes : Flag<["-"], "frewrite-includes">, Group<f_Group>,
   Flags<[CC1Option]>;
 def fno_rewrite_includes : Flag<["-"], "fno-rewrite-includes">, Group<f_Group>;
 
+def frewrite_imports : Flag<["-"], "frewrite-imports">, Group<f_Group>,
+  Flags<[CC1Option]>;
+def fno_rewrite_imports : Flag<["-"], "fno-rewrite-imports">, Group<f_Group>;
+
 def frewrite_map_file : Separate<["-"], "frewrite-map-file">,
                         Group<f_Group>,
                         Flags<[ DriverOption, CC1Option ]>;

include/clang/Frontend/CompilerInstance.h

@@ -140,6 +140,9 @@ class CompilerInstance : public ModuleLoader {
   /// fly as part of this overall compilation action.
   std::map<std::string, std::string> BuiltModules;
 
+  /// Should we delete the BuiltModules when we're done?
+  bool DeleteBuiltModules = true;
+
   /// \brief The location of the module-import keyword for the last module
   /// import. 
   SourceLocation LastModuleImportLoc;

include/clang/Frontend/PreprocessorOutputOptions.h

@@ -24,6 +24,7 @@ public:
   unsigned ShowMacros : 1;         ///< Print macro definitions.
   unsigned ShowIncludeDirectives : 1;  ///< Print includes, imports etc. within preprocessed output.
   unsigned RewriteIncludes : 1;    ///< Preprocess include directives only.
+  unsigned RewriteImports  : 1;    ///< Include contents of transitively-imported modules.
 
 public:
   PreprocessorOutputOptions() {
@@ -35,6 +36,7 @@ public:
     ShowMacros = 0;
     ShowIncludeDirectives = 0;
     RewriteIncludes = 0;
+    RewriteImports = 0;
   }
 };
 

include/clang/Rewrite/Frontend/FrontendActions.h

@@ -11,6 +11,7 @@
 #define LLVM_CLANG_REWRITE_FRONTEND_FRONTENDACTIONS_H
 
 #include "clang/Frontend/FrontendAction.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace clang {
 class FixItRewriter;
@@ -73,7 +74,10 @@ protected:
 };
 
 class RewriteIncludesAction : public PreprocessorFrontendAction {
+  std::shared_ptr<raw_ostream> OutputStream;
+  class RewriteImportsListener;
 protected:
+  bool BeginSourceFileAction(CompilerInstance &CI) override;
   void ExecuteAction() override;
 };
 

lib/Driver/Driver.cpp

@@ -2672,6 +2672,8 @@ Action *Driver::ConstructPhaseAction(Compilation &C, const ArgList &Args,
       OutputTy = Input->getType();
       if (!Args.hasFlag(options::OPT_frewrite_includes,
                         options::OPT_fno_rewrite_includes, false) &&
+          !Args.hasFlag(options::OPT_frewrite_imports,
+                        options::OPT_fno_rewrite_imports, false) &&
           !CCGenDiagnostics)
         OutputTy = types::getPreprocessedType(OutputTy);
       assert(OutputTy != types::TY_INVALID &&

lib/Driver/ToolChains/Clang.cpp

@@ -4204,13 +4204,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 #endif
 
+  bool RewriteImports = Args.hasFlag(options::OPT_frewrite_imports,
+                                     options::OPT_fno_rewrite_imports, false);
+  if (RewriteImports)
+    CmdArgs.push_back("-frewrite-imports");
+
   // Enable rewrite includes if the user's asked for it or if we're generating
   // diagnostics.
   // TODO: Once -module-dependency-dir works with -frewrite-includes it'd be
   // nice to enable this when doing a crashdump for modules as well.
   if (Args.hasFlag(options::OPT_frewrite_includes,
                    options::OPT_fno_rewrite_includes, false) ||
-      (C.isForDiagnostics() && !HaveAnyModules))
+      (C.isForDiagnostics() && (RewriteImports || !HaveAnyModules)))
     CmdArgs.push_back("-frewrite-includes");
 
   // Only allow -traditional or -traditional-cpp outside in preprocessing modes.

lib/Frontend/CompilerInstance.cpp

@@ -667,8 +667,11 @@ void CompilerInstance::clearOutputFiles(bool EraseFiles) {
       llvm::sys::fs::remove(OF.Filename);
   }
   OutputFiles.clear();
-  for (auto &Module : BuiltModules)
-    llvm::sys::fs::remove(Module.second);
+  if (DeleteBuiltModules) {
+    for (auto &Module : BuiltModules)
+      llvm::sys::fs::remove(Module.second);
+    BuiltModules.clear();
+  }
   NonSeekStream.reset();
 }
 
@@ -1928,12 +1931,11 @@ void CompilerInstance::loadModuleFromSource(SourceLocation ImportLoc,
         llvm::MemoryBuffer::getMemBuffer(NullTerminatedSource.c_str()));
 
     Other.BuiltModules = std::move(BuiltModules);
+    Other.DeleteBuiltModules = false;
   };
 
   auto PostBuildStep = [this](CompilerInstance &Other) {
     BuiltModules = std::move(Other.BuiltModules);
-    // Make sure the child build action doesn't delete the .pcms.
-    Other.BuiltModules.clear();
   };
 
   // Build the module, inheriting any modules that we've built locally.

lib/Frontend/CompilerInvocation.cpp

@@ -2501,6 +2501,7 @@ static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
   Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
   Opts.ShowIncludeDirectives = Args.hasArg(OPT_dI);
   Opts.RewriteIncludes = Args.hasArg(OPT_frewrite_includes);
+  Opts.RewriteImports = Args.hasArg(OPT_frewrite_imports);
   Opts.UseLineDirectives = Args.hasArg(OPT_fuse_line_directives);
 }
 

lib/Frontend/Rewrite/FrontendActions.cpp

@@ -18,6 +18,11 @@
 #include "clang/Rewrite/Frontend/ASTConsumers.h"
 #include "clang/Rewrite/Frontend/FixItRewriter.h"
 #include "clang/Rewrite/Frontend/Rewriters.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/Module.h"
+#include "clang/Serialization/ModuleManager.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
@@ -189,27 +194,112 @@ void RewriteTestAction::ExecuteAction() {
   DoRewriteTest(CI.getPreprocessor(), OS.get());
 }
 
-void RewriteIncludesAction::ExecuteAction() {
-  CompilerInstance &CI = getCompilerInstance();
-  std::unique_ptr<raw_ostream> OS =
-      CI.createDefaultOutputFile(true, getCurrentFile());
-  if (!OS) return;
+class RewriteIncludesAction::RewriteImportsListener : public ASTReaderListener {
+  CompilerInstance &CI;
+  std::weak_ptr<raw_ostream> Out;
+
+  llvm::DenseSet<const FileEntry*> Rewritten;
+
+public:
+  RewriteImportsListener(CompilerInstance &CI, std::shared_ptr<raw_ostream> Out)
+      : CI(CI), Out(Out) {}
+
+  void visitModuleFile(StringRef Filename,
+                       serialization::ModuleKind Kind) override {
+    auto *File = CI.getFileManager().getFile(Filename);
+    assert(File && "missing file for loaded module?");
+
+    // Only rewrite each module file once.
+    if (!Rewritten.insert(File).second)
+      return;
+
+    serialization::ModuleFile *MF =
+        CI.getModuleManager()->getModuleManager().lookup(File);
+    assert(File && "missing module file for loaded module?");
+
+    // Not interested in PCH / preambles.
+    if (!MF->isModule())
+      return;
+
+    auto OS = Out.lock();
+    assert(OS && "loaded module file after finishing rewrite action?");
+
+    (*OS) << "#pragma clang module build " << MF->ModuleName << "\n";
+
+    // Rewrite the contents of the module in a separate compiler instance.
+    CompilerInstance Instance(CI.getPCHContainerOperations(),
+                              &CI.getPreprocessor().getPCMCache());
+    Instance.setInvocation(
+        std::make_shared<CompilerInvocation>(CI.getInvocation()));
+    Instance.createDiagnostics(
+        new ForwardingDiagnosticConsumer(CI.getDiagnosticClient()),
+        /*ShouldOwnClient=*/true);
+    Instance.getFrontendOpts().Inputs.clear();
+    Instance.getFrontendOpts().Inputs.emplace_back(
+        Filename, InputKind(InputKind::Unknown, InputKind::Precompiled));
+    // Don't recursively rewrite imports. We handle them all at the top level.
+    Instance.getPreprocessorOutputOpts().RewriteImports = false;
+
+    llvm::CrashRecoveryContext().RunSafelyOnThread([&]() {
+      RewriteIncludesAction Action;
+      Action.OutputStream = OS;
+      Instance.ExecuteAction(Action);
+    });
+
+    (*OS) << "#pragma clang module endbuild /*" << MF->ModuleName << "*/\n";
+  }
+};
+
+bool RewriteIncludesAction::BeginSourceFileAction(CompilerInstance &CI) {
+  if (!OutputStream) {
+    OutputStream = CI.createDefaultOutputFile(true, getCurrentFile());
+    if (!OutputStream)
+      return false;
+  }
+
+  auto &OS = *OutputStream;
 
   // If we're preprocessing a module map, start by dumping the contents of the
   // module itself before switching to the input buffer.
   auto &Input = getCurrentInput();
   if (Input.getKind().getFormat() == InputKind::ModuleMap) {
     if (Input.isFile()) {
-      (*OS) << "# 1 \"";
-      OS->write_escaped(Input.getFile());
-      (*OS) << "\"\n";
+      OS << "# 1 \"";
+      OS.write_escaped(Input.getFile());
+      OS << "\"\n";
     }
-    // FIXME: Include additional information here so that we don't need the
-    // original source files to exist on disk.
-    getCurrentModule()->print(*OS);
-    (*OS) << "#pragma clang module contents\n";
+    getCurrentModule()->print(OS);
+    OS << "#pragma clang module contents\n";
   }
 
-  RewriteIncludesInInput(CI.getPreprocessor(), OS.get(),
-                         CI.getPreprocessorOutputOpts());
+  // If we're rewriting imports, set up a listener to track when we import
+  // module files.
+  if (CI.getPreprocessorOutputOpts().RewriteImports) {
+    CI.createModuleManager();
+    CI.getModuleManager()->addListener(
+        llvm::make_unique<RewriteImportsListener>(CI, OutputStream));
+  }
+
+  return true;
+}
+
+void RewriteIncludesAction::ExecuteAction() {
+  CompilerInstance &CI = getCompilerInstance();
+
+  // If we're rewriting imports, emit the module build output first rather
+  // than switching back and forth (potentially in the middle of a line).
+  if (CI.getPreprocessorOutputOpts().RewriteImports) {
+    std::string Buffer;
+    llvm::raw_string_ostream OS(Buffer);
+
+    RewriteIncludesInInput(CI.getPreprocessor(), &OS,
+                           CI.getPreprocessorOutputOpts());
+
+    (*OutputStream) << OS.str();
+  } else {
+    RewriteIncludesInInput(CI.getPreprocessor(), OutputStream.get(),
+                           CI.getPreprocessorOutputOpts());
+  }
+
+  OutputStream.reset();
 }

lib/FrontendTool/ExecuteCompilerInvocation.cpp

@@ -85,7 +85,8 @@ CreateFrontendBaseAction(CompilerInstance &CI) {
   case PrintDeclContext:       return llvm::make_unique<DeclContextPrintAction>();
   case PrintPreamble:          return llvm::make_unique<PrintPreambleAction>();
   case PrintPreprocessedInput: {
-    if (CI.getPreprocessorOutputOpts().RewriteIncludes)
+    if (CI.getPreprocessorOutputOpts().RewriteIncludes ||
+        CI.getPreprocessorOutputOpts().RewriteImports)
       return llvm::make_unique<RewriteIncludesAction>();
     return llvm::make_unique<PrintPreprocessedAction>();
   }

test/Modules/preprocess-build-diamond.m

@@ -0,0 +1,26 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %S/Inputs %s -E -o %t/diamond.mi -frewrite-imports
+// RUN: FileCheck %s --input-file %t/diamond.mi
+// RUN: %clang_cc1 -fmodules %t/diamond.mi -I. -verify
+
+// CHECK: {{^}}#pragma clang module build diamond_top
+// CHECK: {{^}}module diamond_top {
+// CHECK: {{^}}#pragma clang module contents
+
+// FIXME: @import does not work under -frewrite-includes / -frewrite-imports
+// because we disable it when macro expansion is disabled.
+#include "diamond_bottom.h"
+
+// expected-no-diagnostics
+void test_diamond(int i, float f, double d, char c) {
+  top(&i);
+  left(&f);
+  right(&d);
+  bottom(&c);
+  top_left(&c);
+  left_and_right(&i);
+  struct left_and_right lr;
+  lr.left = 17;
+}
+