From 4151f2d04ad8d829e16b83be734908fc38f977e9 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Mon, 30 Mar 2020 12:58:21 -0700 Subject: [PATCH] Re-land "[FileCollector] Add a method to add a whole directory and it contents." Extend the FileCollector's API with addDirectory which adds a directory and its contents to the VFS mapping. Differential revision: https://reviews.llvm.org/D76671 --- llvm/include/llvm/Support/FileCollector.h | 18 +++++-- llvm/lib/Support/FileCollector.cpp | 56 ++++++++++++-------- llvm/unittests/Support/FileCollectorTest.cpp | 35 ++++++++++++ 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h index 079fe3efab9d..1b6383ef8cc2 100644 --- a/llvm/include/llvm/Support/FileCollector.h +++ b/llvm/include/llvm/Support/FileCollector.h @@ -18,7 +18,7 @@ #include namespace llvm { - +class FileCollectorFileSystem; /// Collects files into a directory and generates a mapping that can be used by /// the VFS. class FileCollector { @@ -26,9 +26,10 @@ public: FileCollector(std::string Root, std::string OverlayRoot); void addFile(const Twine &file); + void addDirectory(const Twine &Dir); /// Write the yaml mapping (for the VFS) to the given file. - std::error_code writeMapping(StringRef mapping_file); + std::error_code writeMapping(StringRef MappingFile); /// Copy the files into the root directory. /// @@ -44,7 +45,7 @@ public: std::shared_ptr Collector); private: - void addFileImpl(StringRef SrcPath); + friend FileCollectorFileSystem; bool markAsSeen(StringRef Path) { if (Path.empty()) @@ -55,10 +56,19 @@ private: bool getRealPath(StringRef SrcPath, SmallVectorImpl &Result); void addFileToMapping(StringRef VirtualPath, StringRef RealPath) { - VFSWriter.addFileMapping(VirtualPath, RealPath); + if (sys::fs::is_directory(VirtualPath)) + VFSWriter.addDirectoryMapping(VirtualPath, RealPath); + else + VFSWriter.addFileMapping(VirtualPath, RealPath); } protected: + void addFileImpl(StringRef SrcPath); + + llvm::vfs::directory_iterator + addDirectoryImpl(const llvm::Twine &Dir, + IntrusiveRefCntPtr FS, std::error_code &EC); + /// Synchronizes adding files. std::mutex Mutex; diff --git a/llvm/lib/Support/FileCollector.cpp b/llvm/lib/Support/FileCollector.cpp index 3a8bc99327bc..f6d72685a8c3 100644 --- a/llvm/lib/Support/FileCollector.cpp +++ b/llvm/lib/Support/FileCollector.cpp @@ -61,13 +61,19 @@ bool FileCollector::getRealPath(StringRef SrcPath, return true; } -void FileCollector::addFile(const Twine &file) { +void FileCollector::addFile(const Twine &File) { std::lock_guard lock(Mutex); - std::string FileStr = file.str(); + std::string FileStr = File.str(); if (markAsSeen(FileStr)) addFileImpl(FileStr); } +void FileCollector::addDirectory(const Twine &Dir) { + assert(sys::fs::is_directory(Dir)); + std::error_code EC; + addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC); +} + void FileCollector::addFileImpl(StringRef SrcPath) { // We need an absolute src path to append to the root. SmallString<256> AbsoluteSrc = SrcPath; @@ -101,6 +107,27 @@ void FileCollector::addFileImpl(StringRef SrcPath) { addFileToMapping(VirtualPath, DstPath); } +llvm::vfs::directory_iterator +FileCollector::addDirectoryImpl(const llvm::Twine &Dir, + IntrusiveRefCntPtr FS, + std::error_code &EC) { + auto It = FS->dir_begin(Dir, EC); + if (EC) + return It; + addFile(Dir); + for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) { + if (It->type() == sys::fs::file_type::regular_file || + It->type() == sys::fs::file_type::directory_file || + It->type() == sys::fs::file_type::symlink_file) { + addFile(It->path()); + } + } + if (EC) + return It; + // Return a new iterator. + return FS->dir_begin(Dir, EC); +} + /// Set the access and modification time for the given file from the given /// status object. static std::error_code @@ -171,7 +198,7 @@ std::error_code FileCollector::copyFiles(bool StopOnError) { return {}; } -std::error_code FileCollector::writeMapping(StringRef mapping_file) { +std::error_code FileCollector::writeMapping(StringRef MappingFile) { std::lock_guard lock(Mutex); VFSWriter.setOverlayDir(OverlayRoot); @@ -179,7 +206,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) { VFSWriter.setUseExternalNames(false); std::error_code EC; - raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text); + raw_fd_ostream os(MappingFile, EC, sys::fs::OF_Text); if (EC) return EC; @@ -188,7 +215,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) { return {}; } -namespace { +namespace llvm { class FileCollectorFileSystem : public vfs::FileSystem { public: @@ -213,22 +240,7 @@ public: llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir, std::error_code &EC) override { - auto It = FS->dir_begin(Dir, EC); - if (EC) - return It; - // Collect everything that's listed in case the user needs it. - Collector->addFile(Dir); - for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) { - if (It->type() == sys::fs::file_type::regular_file || - It->type() == sys::fs::file_type::directory_file || - It->type() == sys::fs::file_type::symlink_file) { - Collector->addFile(It->path()); - } - } - if (EC) - return It; - // Return a new iterator. - return FS->dir_begin(Dir, EC); + return Collector->addDirectoryImpl(Dir, FS, EC); } std::error_code getRealPath(const Twine &Path, @@ -259,7 +271,7 @@ private: std::shared_ptr Collector; }; -} // end anonymous namespace +} // namespace llvm IntrusiveRefCntPtr FileCollector::createCollectorVFS(IntrusiveRefCntPtr BaseFS, diff --git a/llvm/unittests/Support/FileCollectorTest.cpp b/llvm/unittests/Support/FileCollectorTest.cpp index 42a4877dba39..a81ce7130f2f 100644 --- a/llvm/unittests/Support/FileCollectorTest.cpp +++ b/llvm/unittests/Support/FileCollectorTest.cpp @@ -120,6 +120,41 @@ TEST(FileCollectorTest, addFile) { EXPECT_FALSE(FileCollector.hasSeen("/path/to/d")); } +TEST(FileCollectorTest, addDirectory) { + ScopedDir file_root("file_root", true); + + llvm::SmallString<128> aaa = file_root.Path; + llvm::sys::path::append(aaa, "aaa"); + ScopedFile a(aaa.str()); + + llvm::SmallString<128> bbb = file_root.Path; + llvm::sys::path::append(bbb, "bbb"); + ScopedFile b(bbb.str()); + + llvm::SmallString<128> ccc = file_root.Path; + llvm::sys::path::append(ccc, "ccc"); + ScopedFile c(ccc.str()); + + std::string root_fs = std::string(file_root.Path.str()); + TestingFileCollector FileCollector(root_fs, root_fs); + + FileCollector.addDirectory(file_root.Path); + + // Make sure the root is correct. + EXPECT_EQ(FileCollector.Root, root_fs); + + // Make sure we've seen all the added files. + EXPECT_TRUE(FileCollector.hasSeen(a.Path)); + EXPECT_TRUE(FileCollector.hasSeen(b.Path)); + EXPECT_TRUE(FileCollector.hasSeen(c.Path)); + + // Make sure we've only seen the added files. + llvm::SmallString<128> ddd = file_root.Path; + llvm::sys::path::append(ddd, "ddd"); + ScopedFile d(ddd.str()); + EXPECT_FALSE(FileCollector.hasSeen(d.Path)); +} + TEST(FileCollectorTest, copyFiles) { ScopedDir file_root("file_root", true); ScopedFile a(file_root + "/aaa");