[CAS] Support clang include tree

Using clang include tree for dependency scanning and building for all
clang modules and PCHs.

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -139,6 +139,10 @@ SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_textual_detail_get_command_line(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_set_t *
+swiftscan_swift_textual_detail_get_bridging_pch_command_line(
+    swiftscan_module_details_t details);
+
 SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_textual_detail_get_extra_pcm_args(
     swiftscan_module_details_t details);

include/swift/AST/ModuleDependencies.h

@@ -26,8 +26,10 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -128,9 +130,12 @@ class ModuleDependencyInfoStorageBase {
 struct CommonSwiftTextualModuleDependencyDetails {
   CommonSwiftTextualModuleDependencyDetails(
       ArrayRef<StringRef> extraPCMArgs, ArrayRef<StringRef> buildCommandLine,
+      ArrayRef<StringRef> bridgingHeaderBuildCommandLine,
       const std::string &CASFileSystemRootID)
       : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
         buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
+        bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(),
+                                       bridgingHeaderBuildCommandLine.end()),
         CASFileSystemRootID(CASFileSystemRootID) {}
 
   /// To build a PCM to be used by this Swift module, we need to append these
@@ -155,8 +160,14 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// interface.
   std::vector<std::string> buildCommandLine;
 
+  /// The Swift frontend invocation arguments to build bridging header.
+  std::vector<std::string> bridgingHeaderBuildCommandLine;
+
   /// CASID for the Root of CASFS. Empty if CAS is not used.
   std::string CASFileSystemRootID;
+
+  /// CASID for the Root of bridgingHeaderClangIncludeTree. Empty if not used.
+  std::string bridgingHeaderIncludeTreeRoot;
 };
 
 /// Describes the dependencies of a Swift module described by an Swift interface file.
@@ -202,7 +213,7 @@ class SwiftInterfaceModuleDependenciesStorage :
       compiledModuleCandidates(compiledModuleCandidates.begin(),
                                compiledModuleCandidates.end()),
       contextHash(contextHash), isFramework(isFramework),
-      textualModuleDetails(extraPCMArgs, buildCommandLine, RootID),
+      textualModuleDetails(extraPCMArgs, buildCommandLine, {}, RootID),
       moduleCacheKey(ModuleCacheKey)
       {}
 
@@ -218,10 +229,6 @@ class SwiftInterfaceModuleDependenciesStorage :
     textualModuleDetails.buildCommandLine = newCommandLine;
   }
 
-  void updateCASFileSystemID(const std::string &ID) {
-    textualModuleDetails.CASFileSystemRootID = ID;
-  }
-
   void updateModuleCacheKey(const std::string &Key) {
     moduleCacheKey = Key;
   }
@@ -243,11 +250,13 @@ class SwiftSourceModuleDependenciesStorage :
   /// Collection of module imports that were detected to be `@Testable`
   llvm::StringSet<> testableImports;
 
-  SwiftSourceModuleDependenciesStorage(const std::string &RootID,
-                                       ArrayRef<StringRef> buildCommandLine,
-                                       ArrayRef<StringRef> extraPCMArgs)
+  SwiftSourceModuleDependenciesStorage(
+      const std::string &RootID, ArrayRef<StringRef> buildCommandLine,
+      ArrayRef<StringRef> bridgingHeaderBuildCommandLine,
+      ArrayRef<StringRef> extraPCMArgs)
       : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
-        textualModuleDetails(extraPCMArgs, buildCommandLine, RootID),
+        textualModuleDetails(extraPCMArgs, buildCommandLine,
+                             bridgingHeaderBuildCommandLine, RootID),
         testableImports(llvm::StringSet<>()) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
@@ -262,8 +271,9 @@ class SwiftSourceModuleDependenciesStorage :
     textualModuleDetails.buildCommandLine = newCommandLine;
   }
 
-  void updateCASFileSystemID(const std::string &ID) {
-    textualModuleDetails.CASFileSystemRootID = ID;
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    textualModuleDetails.bridgingHeaderBuildCommandLine = newCommandLine;
   }
 
   void addTestableImport(ImportPath::Module module) {
@@ -341,6 +351,9 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
   /// CASID for the Root of CASFS. Empty if CAS is not used.
   std::string CASFileSystemRootID;
 
+  /// CASID for the Root of ClangIncludeTree. Empty if not used.
+  std::string clangIncludeTreeRoot;
+
   /// The cache key for the produced module.
   std::string moduleCacheKey;
 
@@ -352,6 +365,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
       const std::vector<std::string> &fileDependencies,
       const std::vector<std::string> &capturedPCMArgs,
       const std::string &CASFileSystemRootID,
+      const std::string &clangIncludeTreeRoot,
       const std::string &ModuleCacheKey
   ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang),
       pcmOutputPath(pcmOutputPath),
@@ -361,6 +375,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
       fileDependencies(fileDependencies),
       capturedPCMArgs(capturedPCMArgs),
       CASFileSystemRootID(CASFileSystemRootID),
+      clangIncludeTreeRoot(clangIncludeTreeRoot),
       moduleCacheKey(ModuleCacheKey) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
@@ -473,10 +488,12 @@ class ModuleDependencyInfo {
   static ModuleDependencyInfo
   forSwiftSourceModule(const std::string &CASFileSystemRootID,
                        ArrayRef<StringRef> buildCommands,
+                       ArrayRef<StringRef> bridgingHeaderBuildCommands,
                        ArrayRef<StringRef> extraPCMArgs) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftSourceModuleDependenciesStorage>(
-            CASFileSystemRootID, buildCommands, extraPCMArgs));
+            CASFileSystemRootID, buildCommands, bridgingHeaderBuildCommands,
+            extraPCMArgs));
   }
 
   /// Describe the module dependencies for a Clang module that can be
@@ -489,12 +506,13 @@ class ModuleDependencyInfo {
       const std::vector<std::string> &fileDependencies,
       const std::vector<std::string> &capturedPCMArgs,
       const std::string &CASFileSystemRootID,
+      const std::string &clangIncludeTreeRoot,
       const std::string &moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<ClangModuleDependencyStorage>(
           pcmOutputPath, moduleMapFile, contextHash,
           nonPathCommandLine, fileDependencies, capturedPCMArgs,
-          CASFileSystemRootID, moduleCacheKey));
+          CASFileSystemRootID, clangIncludeTreeRoot,  moduleCacheKey));
   }
 
   /// Describe a placeholder dependency swift module.
@@ -593,13 +611,17 @@ class ModuleDependencyInfo {
     llvm_unreachable("Unexpected type");
   }
 
-  void updateCASFileSystemID(const std::string &ID) {
-    if (isSwiftInterfaceModule())
-      return cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())
-          ->updateCASFileSystemID(ID);
-    else if (isSwiftSourceModule())
+  std::vector<std::string> getBridgingHeaderCommandline() const {
+    if (auto *detail = getAsSwiftSourceModule())
+      return detail->textualModuleDetails.bridgingHeaderBuildCommandLine;
+    return {};
+  }
+
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    if (isSwiftSourceModule())
       return cast<SwiftSourceModuleDependenciesStorage>(storage.get())
-          ->updateCASFileSystemID(ID);
+          ->updateBridgingHeaderCommandLine(newCommandLine);
     llvm_unreachable("Unexpected type");
   }
 
@@ -695,6 +717,12 @@ class ModuleDependencyInfo {
   /// Get CAS Filesystem RootID.
   Optional<std::string> getCASFSRootID() const;
 
+  /// Get Clang Include Tree ID.
+  Optional<std::string> getClangIncludeTree() const;
+
+  /// Get bridging header Include Tree ID.
+  Optional<std::string> getBridgingHeaderIncludeTree() const;
+
   /// Get module output path.
   std::string getModuleOutputPath() const;
 
@@ -711,6 +739,9 @@ class ModuleDependencyInfo {
   void addBridgingModuleDependency(StringRef module,
                                    llvm::StringSet<> &alreadyAddedModules);
 
+  /// Add bridging header include tree.
+  void addBridgingHeaderIncludeTree(StringRef ID);
+
   /// Collect a map from a secondary module name to a list of cross-import
   /// overlays, when this current module serves as the primary module.
   llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
@@ -773,6 +804,12 @@ class SwiftDependencyScanningService {
   /// CachingOnDiskFileSystem for dependency tracking.
   llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
 
+  /// If use clang include tree.
+  bool UseClangIncludeTree = false;
+
+  /// CAS ObjectStore Instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+
   /// The common dependencies that is needed for every swift compiler instance.
   std::vector<std::string> CommonDependencyFiles;
 
@@ -826,7 +863,7 @@ class SwiftDependencyScanningService {
     return getCacheForScanningContextHash(scanningContextHash)->alreadySeenClangModules;
   }
 
-  bool usingCachingFS() const { return (bool)CacheFS; }
+  bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
 
   llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
     assert(CacheFS && "Expect CachingOnDiskFileSystem");
@@ -841,9 +878,13 @@ class SwiftDependencyScanningService {
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
-    if (CacheFS)
+    if (usingCachingFS())
       return CacheFS->createProxyFS();
 
+    if (UseClangIncludeTree)
+      return llvm::cas::createCASProvidingFileSystem(
+          CAS, llvm::vfs::createPhysicalFileSystem());
+
     return llvm::vfs::createPhysicalFileSystem();
   }
 

include/swift/Basic/LangOptions.h

@@ -890,6 +890,9 @@ namespace swift {
     /// built and provided to the compiler invocation.
     bool DisableImplicitClangModules = false;
 
+    /// Enable ClangIncludeTree for explicit module builds.
+    bool UseClangIncludeTree = false;
+
     /// Return a hash code of any components from these options that should
     /// contribute to a Swift Bridging PCH hash.
     llvm::hash_code getPCHHashComponents() const {

include/swift/ClangImporter/ClangImporter.h

@@ -54,6 +54,7 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
   struct ModuleDeps;
+  struct TranslationUnitDeps;
   using ModuleDepsGraph = std::vector<ModuleDeps>;
 }
 }
@@ -431,6 +432,10 @@ class ClangImporter final : public ClangModuleLoader {
       ModuleDependenciesCache &cache,
       const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies);
 
+  void recordBridgingHeaderOptions(
+      ModuleDependencyInfo &MDI,
+      const clang::tooling::dependencies::TranslationUnitDeps &deps);
+
   Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate,

include/swift/DependencyScan/DependencyScanImpl.h

@@ -86,6 +86,9 @@ typedef struct {
   /// Options to the compile command required to build this module interface
   swiftscan_string_set_t *command_line;
 
+  /// Options to the compile command required to build bridging header.
+  swiftscan_string_set_t *bridging_pch_command_line;
+
   /// To build a PCM to be used by this Swift module, we need to append these
   /// arguments to the generic PCM build arguments reported from the dependency
   /// graph.
@@ -100,6 +103,9 @@ typedef struct {
   /// The CASID for CASFileSystemRoot
   swiftscan_string_ref_t cas_fs_root_id;
 
+  /// The CASID for bridging header include tree
+  swiftscan_string_ref_t bridging_header_include_tree;
+
   /// ModuleCacheKey
   swiftscan_string_ref_t module_cache_key;
 } swiftscan_swift_textual_details_t;
@@ -152,6 +158,9 @@ typedef struct {
   /// The CASID for CASFileSystemRoot
   swiftscan_string_ref_t cas_fs_root_id;
 
+  /// The CASID for CASFileSystemRoot
+  swiftscan_string_ref_t clang_include_tree;
+
   /// ModuleCacheKey
   swiftscan_string_ref_t module_cache_key;
 } swiftscan_clang_details_t;

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -142,6 +142,7 @@ using SwiftInterfaceModuleDetailsLayout =
                    FileIDArrayIDField,                  // bridgingModuleDependencies
                    DependencyIDArrayIDField,            // swiftOverlayDependencies
                    IdentifierIDField,                   // CASFileSystemRootID
+                   IdentifierIDField,                   // bridgingHeaderIncludeTree
                    IdentifierIDField                    // moduleCacheKey
                    >;
 
@@ -154,7 +155,9 @@ using SwiftSourceModuleDetailsLayout =
                    FileIDArrayIDField,               // bridgingModuleDependencies
                    DependencyIDArrayIDField,         // swiftOverlayDependencies
                    IdentifierIDField,                // CASFileSystemRootID
-                   FlagIDArrayIDField                // buildCommandLine
+                   IdentifierIDField,                // bridgingHeaderIncludeTree
+                   FlagIDArrayIDField,               // buildCommandLine
+                   FlagIDArrayIDField                // bridgingHeaderBuildCommandLine
                    >;
 
 using SwiftBinaryModuleDetailsLayout =
@@ -182,6 +185,7 @@ using ClangModuleDetailsLayout =
                    FileIDArrayIDField,        // fileDependencies
                    FlagIDArrayIDField,        // capturedPCMArgs
                    IdentifierIDField,         // CASFileSystemRootID
+                   IdentifierIDField,         // clangIncludeTreeRoot
                    IdentifierIDField          // moduleCacheKey
                    >;
 } // namespace graph_block

include/swift/Frontend/CachingUtils.h

@@ -19,6 +19,7 @@
 #include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/CAS/CASReference.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <memory>
 
@@ -54,6 +55,10 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
                                       StringRef CorrespondingInput,
                                       file_types::ID OutputKind);
 
+llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
+createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
+                    ArrayRef<std::string> IncludeTreeRoots);
+
 namespace cas {
 /// Helper class to manage CAS/Caching from libSwiftScan C APIs.
 class CachingTool {

include/swift/Frontend/FrontendOptions.h

@@ -129,7 +129,10 @@ class FrontendOptions {
   std::string CASPath;
 
   /// CASFS Root.
-  std::string CASFSRootID;
+  std::vector<std::string> CASFSRootIDs;
+
+  /// Clang Include Trees.
+  std::vector<std::string> ClangIncludeTrees;
 
   /// Number of retry opening an input file if the previous opening returns
   /// bad file descriptor error.

include/swift/Option/Options.td

@@ -1833,6 +1833,10 @@ def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
   Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
   HelpText<"Cache Key for bridging header pch">;
 
+def clang_include_tree: Flag<["-"], "clang-include-tree">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Use clang include tree">;
+
 
 // END ONLY SUPPORTED IN NEW DRIVER
 
@@ -1844,6 +1848,10 @@ def cas_fs: Separate<["-"], "cas-fs">,
   Flags<[FrontendOption, NoDriverOption]>,
   HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
 
+def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
+
 def load_plugin_library:
   Separate<["-"], "load-plugin-library">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,