Merge commit 'c1aa61938dd2' from apple/stable/20200108 into swift/master

Author: git apple-llvm automerger

lldb/include/lldb/Utility/Reproducer.h

@@ -98,6 +98,8 @@ class FileProvider : public Provider<FileProvider> {
     return m_collector;
   }
 
+  void recordInterestingDirectory(const llvm::Twine &dir);
+
   void Keep() override {
     auto mapping = GetRoot().CopyByAppendingPathComponent(Info::file);
     // Temporary files that are removed during execution can cause copy errors.

lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp

@@ -321,6 +321,11 @@ void WorkingDirectoryProvider::Keep() {
   os << m_cwd << "\n";
 }
 
+void FileProvider::recordInterestingDirectory(const llvm::Twine &dir) {
+  if (m_collector)
+    m_collector->addDirectory(dir);
+}
+
 void ProviderBase::anchor() {}
 char CommandProvider::ID = 0;
 char FileProvider::ID = 0;

lldb/source/Utility/Reproducer.cpp

@@ -0,0 +1,11 @@
+# REQUIRES: system-darwin
+# Ensure that the reproducers captures the whole dSYM bundle.
+
+# RUN: rm -rf %t.repro
+# RUN: %clang_host %S/Inputs/simple.c -g -o %t.out
+# RUN: touch %t.out.dSYM/foo.bar
+
+# RUN: %lldb -x -b --capture --capture-path %t.repro %t.out -o 'b main' -o 'run' -o 'reproducer generate'
+
+# RUN: %lldb -b -o 'reproducer dump -p files -f %t.repro' | FileCheck %s --check-prefix FILES
+# FILES: foo.bar

lldb/test/Shell/Reproducer/TestDSYM.test

@@ -18,17 +18,18 @@
 #include <mutex>
 
 namespace llvm {
-
+class FileCollectorFileSystem;
 /// Collects files into a directory and generates a mapping that can be used by
 /// the VFS.
 class FileCollector {
 public:
   FileCollector(std::string Root, std::string OverlayRoot);
 
   void addFile(const Twine &file);
+  void addDirectory(const Twine &Dir);
 
   /// Write the yaml mapping (for the VFS) to the given file.
-  std::error_code writeMapping(StringRef mapping_file);
+  std::error_code writeMapping(StringRef MappingFile);
 
   /// Copy the files into the root directory.
   ///
@@ -44,7 +45,7 @@ class FileCollector {
                      std::shared_ptr<FileCollector> Collector);
 
 private:
-  void addFileImpl(StringRef SrcPath);
+  friend FileCollectorFileSystem;
 
   bool markAsSeen(StringRef Path) {
     if (Path.empty())
@@ -55,10 +56,19 @@ class FileCollector {
   bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result);
 
   void addFileToMapping(StringRef VirtualPath, StringRef RealPath) {
-    VFSWriter.addFileMapping(VirtualPath, RealPath);
+    if (sys::fs::is_directory(VirtualPath))
+      VFSWriter.addDirectoryMapping(VirtualPath, RealPath);
+    else
+      VFSWriter.addFileMapping(VirtualPath, RealPath);
   }
 
 protected:
+  void addFileImpl(StringRef SrcPath);
+
+  llvm::vfs::directory_iterator
+  addDirectoryImpl(const llvm::Twine &Dir,
+                   IntrusiveRefCntPtr<vfs::FileSystem> FS, std::error_code &EC);
+
   /// Synchronizes adding files.
   std::mutex Mutex;
 

llvm/include/llvm/Support/FileCollector.h

@@ -506,10 +506,12 @@ getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
 
 struct YAMLVFSEntry {
   template <typename T1, typename T2>
-  YAMLVFSEntry(T1 &&VPath, T2 &&RPath)
-      : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)) {}
+  YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false)
+      : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)),
+        IsDirectory(IsDirectory) {}
   std::string VPath;
   std::string RPath;
+  bool IsDirectory = false;
 };
 
 class VFSFromYamlDirIterImpl;
@@ -781,10 +783,13 @@ class YAMLVFSWriter {
   Optional<bool> UseExternalNames;
   std::string OverlayDir;
 
+  void addEntry(StringRef VirtualPath, StringRef RealPath, bool IsDirectory);
+
 public:
   YAMLVFSWriter() = default;
 
   void addFileMapping(StringRef VirtualPath, StringRef RealPath);
+  void addDirectoryMapping(StringRef VirtualPath, StringRef RealPath);
 
   void setCaseSensitivity(bool CaseSensitive) {
     IsCaseSensitive = CaseSensitive;

llvm/include/llvm/Support/VirtualFileSystem.h

@@ -61,13 +61,19 @@ bool FileCollector::getRealPath(StringRef SrcPath,
   return true;
 }
 
-void FileCollector::addFile(const Twine &file) {
+void FileCollector::addFile(const Twine &File) {
   std::lock_guard<std::mutex> lock(Mutex);
-  std::string FileStr = file.str();
+  std::string FileStr = File.str();
   if (markAsSeen(FileStr))
     addFileImpl(FileStr);
 }
 
+void FileCollector::addDirectory(const Twine &Dir) {
+  assert(sys::fs::is_directory(Dir));
+  std::error_code EC;
+  addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC);
+}
+
 void FileCollector::addFileImpl(StringRef SrcPath) {
   // We need an absolute src path to append to the root.
   SmallString<256> AbsoluteSrc = SrcPath;
@@ -101,6 +107,27 @@ void FileCollector::addFileImpl(StringRef SrcPath) {
   addFileToMapping(VirtualPath, DstPath);
 }
 
+llvm::vfs::directory_iterator
+FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
+                                IntrusiveRefCntPtr<vfs::FileSystem> FS,
+                                std::error_code &EC) {
+  auto It = FS->dir_begin(Dir, EC);
+  if (EC)
+    return It;
+  addFile(Dir);
+  for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
+    if (It->type() == sys::fs::file_type::regular_file ||
+        It->type() == sys::fs::file_type::directory_file ||
+        It->type() == sys::fs::file_type::symlink_file) {
+      addFile(It->path());
+    }
+  }
+  if (EC)
+    return It;
+  // Return a new iterator.
+  return FS->dir_begin(Dir, EC);
+}
+
 /// Set the access and modification time for the given file from the given
 /// status object.
 static std::error_code
@@ -171,15 +198,15 @@ std::error_code FileCollector::copyFiles(bool StopOnError) {
   return {};
 }
 
-std::error_code FileCollector::writeMapping(StringRef mapping_file) {
+std::error_code FileCollector::writeMapping(StringRef MappingFile) {
   std::lock_guard<std::mutex> lock(Mutex);
 
   VFSWriter.setOverlayDir(OverlayRoot);
   VFSWriter.setCaseSensitivity(isCaseSensitivePath(OverlayRoot));
   VFSWriter.setUseExternalNames(false);
 
   std::error_code EC;
-  raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
+  raw_fd_ostream os(MappingFile, EC, sys::fs::OF_Text);
   if (EC)
     return EC;
 
@@ -188,7 +215,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   return {};
 }
 
-namespace {
+namespace llvm {
 
 class FileCollectorFileSystem : public vfs::FileSystem {
 public:
@@ -213,22 +240,7 @@ class FileCollectorFileSystem : public vfs::FileSystem {
 
   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
                                           std::error_code &EC) override {
-    auto It = FS->dir_begin(Dir, EC);
-    if (EC)
-      return It;
-    // Collect everything that's listed in case the user needs it.
-    Collector->addFile(Dir);
-    for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
-      if (It->type() == sys::fs::file_type::regular_file ||
-          It->type() == sys::fs::file_type::directory_file ||
-          It->type() == sys::fs::file_type::symlink_file) {
-        Collector->addFile(It->path());
-      }
-    }
-    if (EC)
-      return It;
-    // Return a new iterator.
-    return FS->dir_begin(Dir, EC);
+    return Collector->addDirectoryImpl(Dir, FS, EC);
   }
 
   std::error_code getRealPath(const Twine &Path,
@@ -259,7 +271,7 @@ class FileCollectorFileSystem : public vfs::FileSystem {
   std::shared_ptr<FileCollector> Collector;
 };
 
-} // end anonymous namespace
+} // namespace llvm
 
 IntrusiveRefCntPtr<vfs::FileSystem>
 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,

llvm/lib/Support/FileCollector.cpp

@@ -1894,11 +1894,21 @@ UniqueID vfs::getNextVirtualUniqueID() {
   return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
 }
 
-void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
+void YAMLVFSWriter::addEntry(StringRef VirtualPath, StringRef RealPath,
+                             bool IsDirectory) {
   assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
   assert(sys::path::is_absolute(RealPath) && "real path not absolute");
   assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
-  Mappings.emplace_back(VirtualPath, RealPath);
+  Mappings.emplace_back(VirtualPath, RealPath, IsDirectory);
+}
+
+void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
+  addEntry(VirtualPath, RealPath, /*IsDirectory=*/false);
+}
+
+void YAMLVFSWriter::addDirectoryMapping(StringRef VirtualPath,
+                                        StringRef RealPath) {
+  addEntry(VirtualPath, RealPath, /*IsDirectory=*/true);
 }
 
 namespace {
@@ -1999,7 +2009,10 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
 
   if (!Entries.empty()) {
     const YAMLVFSEntry &Entry = Entries.front();
-    startDirectory(path::parent_path(Entry.VPath));
+    bool first_entry_is_directory = Entry.IsDirectory;
+    StringRef Dir = first_entry_is_directory ? StringRef(Entry.VPath)
+                                             : path::parent_path(Entry.VPath);
+    startDirectory(Dir);
 
     StringRef RPath = Entry.RPath;
     if (UseOverlayRelative) {
@@ -2009,13 +2022,18 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
       RPath = RPath.slice(OverlayDirLen, RPath.size());
     }
 
-    writeEntry(path::filename(Entry.VPath), RPath);
+    if (!first_entry_is_directory)
+      writeEntry(path::filename(Entry.VPath), RPath);
 
     for (const auto &Entry : Entries.slice(1)) {
-      StringRef Dir = path::parent_path(Entry.VPath);
-      if (Dir == DirStack.back())
-        OS << ",\n";
-      else {
+      StringRef Dir = Entry.IsDirectory ? StringRef(Entry.VPath)
+                                        : path::parent_path(Entry.VPath);
+      if (Dir == DirStack.back()) {
+        if (!first_entry_is_directory) {
+          OS << ",\n";
+          first_entry_is_directory = false;
+        }
+      } else {
         while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
           OS << "\n";
           endDirectory();

llvm/lib/Support/VirtualFileSystem.cpp

@@ -120,6 +120,41 @@ TEST(FileCollectorTest, addFile) {
   EXPECT_FALSE(FileCollector.hasSeen("/path/to/d"));
 }
 
+TEST(FileCollectorTest, addDirectory) {
+  ScopedDir file_root("file_root", true);
+
+  llvm::SmallString<128> aaa = file_root.Path;
+  llvm::sys::path::append(aaa, "aaa");
+  ScopedFile a(aaa.str());
+
+  llvm::SmallString<128> bbb = file_root.Path;
+  llvm::sys::path::append(bbb, "bbb");
+  ScopedFile b(bbb.str());
+
+  llvm::SmallString<128> ccc = file_root.Path;
+  llvm::sys::path::append(ccc, "ccc");
+  ScopedFile c(ccc.str());
+
+  std::string root_fs = std::string(file_root.Path.str());
+  TestingFileCollector FileCollector(root_fs, root_fs);
+
+  FileCollector.addDirectory(file_root.Path);
+
+  // Make sure the root is correct.
+  EXPECT_EQ(FileCollector.Root, root_fs);
+
+  // Make sure we've seen all the added files.
+  EXPECT_TRUE(FileCollector.hasSeen(a.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(b.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(c.Path));
+
+  // Make sure we've only seen the added files.
+  llvm::SmallString<128> ddd = file_root.Path;
+  llvm::sys::path::append(ddd, "ddd");
+  ScopedFile d(ddd.str());
+  EXPECT_FALSE(FileCollector.hasSeen(d.Path));
+}
+
 TEST(FileCollectorTest, copyFiles) {
   ScopedDir file_root("file_root", true);
   ScopedFile a(file_root + "/aaa");

llvm/unittests/Support/FileCollectorTest.cpp

@@ -2189,3 +2189,53 @@ TEST_F(VFSFromYAMLTest, WorkingDirectoryFallthroughInvalid) {
   Status = FS->status("foo/a");
   ASSERT_TRUE(Status.getError());
 }
+
+TEST_F(VFSFromYAMLTest, YAMLVFSWriterTest) {
+  ScopedDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
+  ScopedDir _a(TestDirectory + "/a");
+  ScopedFile _ab(TestDirectory + "/a/b", "");
+  ScopedDir _c(TestDirectory + "/c");
+  ScopedFile _cd(TestDirectory + "/c/d", "");
+  ScopedDir _e(TestDirectory + "/e");
+  ScopedDir _ef(TestDirectory + "/e/f");
+  ScopedDir _g(TestDirectory + "/g");
+  ScopedFile _h(TestDirectory + "/h", "");
+
+  vfs::YAMLVFSWriter VFSWriter;
+  VFSWriter.addDirectoryMapping(_a.Path, "//root/a");
+  VFSWriter.addFileMapping(_ab.Path, "//root/a/b");
+  VFSWriter.addFileMapping(_cd.Path, "//root/c/d");
+  VFSWriter.addDirectoryMapping(_e.Path, "//root/e");
+  VFSWriter.addDirectoryMapping(_ef.Path, "//root/e/f");
+  VFSWriter.addFileMapping(_g.Path, "//root/g");
+  VFSWriter.addDirectoryMapping(_h.Path, "//root/h");
+
+  std::string Buffer;
+  raw_string_ostream OS(Buffer);
+  VFSWriter.write(OS);
+  OS.flush();
+
+  IntrusiveRefCntPtr<ErrorDummyFileSystem> Lower(new ErrorDummyFileSystem());
+  Lower->addDirectory("//root/");
+  Lower->addDirectory("//root/a");
+  Lower->addRegularFile("//root/a/b");
+  Lower->addDirectory("//root/b");
+  Lower->addDirectory("//root/c");
+  Lower->addRegularFile("//root/c/d");
+  Lower->addDirectory("//root/e");
+  Lower->addDirectory("//root/e/f");
+  Lower->addDirectory("//root/g");
+  Lower->addRegularFile("//root/h");
+
+  IntrusiveRefCntPtr<vfs::FileSystem> FS = getFromYAMLRawString(Buffer, Lower);
+  ASSERT_TRUE(FS.get() != nullptr);
+
+  EXPECT_TRUE(FS->exists(_a.Path));
+  EXPECT_TRUE(FS->exists(_ab.Path));
+  EXPECT_TRUE(FS->exists(_c.Path));
+  EXPECT_TRUE(FS->exists(_cd.Path));
+  EXPECT_TRUE(FS->exists(_e.Path));
+  EXPECT_TRUE(FS->exists(_ef.Path));
+  EXPECT_TRUE(FS->exists(_g.Path));
+  EXPECT_TRUE(FS->exists(_h.Path));
+}