Merge pull request #1375 from artemcm/TransitivePCHDepsOfBinaryModules

[Explicit Module Builds] Add support for header dependencies of binary Swift module dependencies

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -132,6 +132,8 @@ typedef struct {
   (*swiftscan_swift_binary_detail_get_module_doc_path)(swiftscan_module_details_t);
   swiftscan_string_ref_t
   (*swiftscan_swift_binary_detail_get_module_source_info_path)(swiftscan_module_details_t);
+  swiftscan_string_set_t *
+  (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t);
   bool
   (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t);
 

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -234,6 +234,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
     for dependencyModule in swiftDependencyArtifacts {
       inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path,
                                      type: .swiftModule))
+
+      for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] {
+        commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"])
+        commandLine.appendPath(VirtualPath.lookup(headerDep.path))
+        inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch))
+      }
     }
 
     // Clang module dependencies are specified on the command line explicitly
@@ -301,6 +307,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         swiftDependencyArtifacts.append(
           SwiftModuleArtifactInfo(name: dependencyId.moduleName,
                                   modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
+                                  headerDependencies: prebuiltModuleDetails.headerDependencyPaths,
                                   isFramework: isFramework))
       case .swiftPlaceholder:
         fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -146,16 +146,21 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable {
   /// The path to the .swiftSourceInfo file.
   public var moduleSourceInfoPath: TextualVirtualPath?
 
+  /// The paths to the binary module's header dependencies
+  public var headerDependencyPaths: [TextualVirtualPath]?
+
   /// A flag to indicate whether or not this module is a framework.
   public var isFramework: Bool?
 
   public init(compiledModulePath: TextualVirtualPath,
               moduleDocPath: TextualVirtualPath? = nil,
               moduleSourceInfoPath: TextualVirtualPath? = nil,
+              headerDependencies: [TextualVirtualPath]? = nil,
               isFramework: Bool) throws {
     self.compiledModulePath = compiledModulePath
     self.moduleDocPath = moduleDocPath
     self.moduleSourceInfoPath = moduleSourceInfoPath
+    self.headerDependencyPaths = headerDependencies
     self.isFramework = isFramework
   }
 }

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -132,16 +132,23 @@ public class InterModuleDependencyOracle {
 
   @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool {
     guard let swiftScan = swiftScanLibInstance else {
-      fatalError("Attempting to reset scanner cache with no scanner instance.")
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsScannerDiagnostics
+  }
+
+  @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
     }
-    return swiftScan.supportsScannerDiagnostics()
+    return swiftScan.supportsBinaryModuleHeaderDependencies
   }
 
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")
     }
-    guard swiftScan.supportsScannerDiagnostics() else {
+    guard swiftScan.supportsScannerDiagnostics else {
       return nil
     }
     let diags = try swiftScan.queryScannerDiagnostics()

Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift

@@ -24,15 +24,19 @@
   public let docPath: TextualVirtualPath?
   /// The path for the module's .swiftsourceinfo file
   public let sourceInfoPath: TextualVirtualPath?
+  /// Header dependencies of this module
+  public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]?
   /// A flag to indicate whether this module is a framework
   public let isFramework: Bool
 
   init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil,
-       sourceInfoPath: TextualVirtualPath? = nil, isFramework: Bool = false) {
+       sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil,
+       isFramework: Bool = false) {
     self.moduleName = name
     self.modulePath = modulePath
     self.docPath = docPath
     self.sourceInfoPath = sourceInfoPath
+    self.prebuiltHeaderDependencyPaths = headerDependencies
     self.isFramework = isFramework
   }
 }

Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

@@ -195,7 +195,7 @@ private extension SwiftScan {
 
     // Decode all dependencies of this module
     let swiftOverlayDependencies: [ModuleDependencyId]?
-    if supportsSeparateSwiftOverlayDependencies(),
+    if supportsSeparateSwiftOverlayDependencies,
        let encodedOverlayDepsRef = api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies(moduleDetailsRef) {
       let encodedOverlayDependencies = try toSwiftStringArray(encodedOverlayDepsRef.pointee)
       swiftOverlayDependencies =
@@ -228,6 +228,14 @@ private extension SwiftScan {
       try getOptionalPathDetail(from: moduleDetailsRef,
                                 using: api.swiftscan_swift_binary_detail_get_module_source_info_path)
 
+    let headerDependencies: [TextualVirtualPath]?
+    if supportsBinaryModuleHeaderDependencies {
+      headerDependencies = try getOptionalPathArrayDetail(from: moduleDetailsRef,
+                                                          using: api.swiftscan_swift_binary_detail_get_header_dependencies)
+    } else {
+      headerDependencies = nil
+    }
+
     let isFramework: Bool
     if hasBinarySwiftModuleIsFramework {
       isFramework = api.swiftscan_swift_binary_detail_get_is_framework(moduleDetailsRef)
@@ -238,6 +246,7 @@ private extension SwiftScan {
     return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath,
                                                   moduleDocPath: moduleDocPath,
                                                   moduleSourceInfoPath: moduleSourceInfoPath,
+                                                  headerDependencies: headerDependencies,
                                                   isFramework: isFramework)
   }
 

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -250,34 +250,39 @@ internal extension swiftscan_diagnostic_severity_t {
     api.swiftscan_clang_detail_get_captured_pcm_args != nil
   }
 
-  func serializeScannerCache(to path: AbsolutePath) {
-    api.swiftscan_scanner_cache_serialize(scanner,
-                                          path.description.cString(using: String.Encoding.utf8))
+  @_spi(Testing) public var supportsBinaryModuleHeaderDependencies : Bool {
+    return api.swiftscan_swift_binary_detail_get_header_dependencies != nil
   }
 
-  func loadScannerCache(from path: AbsolutePath) -> Bool {
-    return api.swiftscan_scanner_cache_load(scanner,
-                                            path.description.cString(using: String.Encoding.utf8))
+  @_spi(Testing) public var supportsStringDispose : Bool {
+    return api.swiftscan_string_dispose != nil
   }
 
-  func resetScannerCache() {
-    api.swiftscan_scanner_cache_reset(scanner)
-  }
 
-  @_spi(Testing) public func supportsSeparateSwiftOverlayDependencies() -> Bool {
+  @_spi(Testing) public var supportsSeparateSwiftOverlayDependencies : Bool {
     return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil
   }
 
-  @_spi(Testing) public func supportsScannerDiagnostics() -> Bool {
+  @_spi(Testing) public var supportsScannerDiagnostics : Bool {
     return api.swiftscan_scanner_diagnostics_query != nil &&
            api.swiftscan_scanner_diagnostics_reset != nil &&
            api.swiftscan_diagnostic_get_message != nil &&
            api.swiftscan_diagnostic_get_severity != nil &&
            api.swiftscan_diagnostics_set_dispose != nil
   }
 
-  @_spi(Testing) public func supportsStringDispose() -> Bool {
-    return api.swiftscan_string_dispose != nil
+  func serializeScannerCache(to path: AbsolutePath) {
+    api.swiftscan_scanner_cache_serialize(scanner,
+                                          path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func loadScannerCache(from path: AbsolutePath) -> Bool {
+    return api.swiftscan_scanner_cache_load(scanner,
+                                            path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func resetScannerCache() {
+    api.swiftscan_scanner_cache_reset(scanner)
   }
 
   @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
@@ -427,6 +432,10 @@ private extension swiftscan_functions_t {
     self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies =
       try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies")
 
+    // Header dependencies of binary modules
+    self.swiftscan_swift_binary_detail_get_header_dependencies =
+      try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -905,6 +905,82 @@ final class ExplicitModuleBuildTests: XCTestCase {
     }
   }
 
+  func testExplicitModuleBuildEndToEndWithBinaryHeaderDeps() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let PCHPath = path.appending(component: "PCH")
+      try localFileSystem.createDirectory(PCHPath)
+      let FooInstallPath = path.appending(component: "Foo")
+      try localFileSystem.createDirectory(FooInstallPath)
+      let foo = path.appending(component: "foo.swift")
+      try localFileSystem.writeFileContents(foo) {
+        $0 <<< "extension Profiler {"
+        $0 <<< "    public static let count: Int = 42"
+        $0 <<< "}"
+      }
+      let fooHeader = path.appending(component: "foo.h")
+      try localFileSystem.writeFileContents(fooHeader) {
+        $0 <<< "struct Profiler { void* ptr; };"
+      }
+      let main = path.appending(component: "main.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import Foo"
+      }
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+
+      var fooBuildDriver = try Driver(args: ["swiftc",
+                                             "-explicit-module-build",
+                                             "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                             "-working-directory", path.nativePathString(escaped: true),
+                                             foo.nativePathString(escaped: true),
+                                             "-emit-module", "-wmo", "-module-name", "Foo",
+                                             "-emit-module-path", FooInstallPath.nativePathString(escaped: true),
+                                             "-import-objc-header", fooHeader.nativePathString(escaped: true),
+                                             "-pch-output-dir", PCHPath.nativePathString(escaped: true),
+                                             FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)]
+                                      + sdkArgumentsForTesting,
+                                      env: ProcessEnv.vars)
+
+      // Ensure this tooling supports this functionality
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+      guard try dependencyOracle.supportsBinaryModuleHeaderDependencies() else {
+        throw XCTSkip("libSwiftScan does not support binary module header dependencies.")
+      }
+
+      let fooJobs = try fooBuildDriver.planBuild()
+      try fooBuildDriver.run(jobs: fooJobs)
+      XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
+
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", FooInstallPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-emit-module", "-emit-module-path",
+                                     path.appending(component: "testEMBETEWBHD.swiftmodule").nativePathString(escaped: true),
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      let jobs = try driver.planBuild()
+      let compileJob = try XCTUnwrap(jobs.first(where: { $0.description == "Compiling main main.swift" }))
+
+      // Ensure the header dependency of Foo shows up on client compile commands
+      XCTAssertTrue(compileJob.commandLine.contains(subsequence: [.flag("-Xcc"),
+                                                                  .flag("-include-pch"),
+                                                                  .flag("-Xcc"),
+                                                                  .path(.absolute(PCHPath.appending(component: "foo.pch")))]))
+      try driver.run(jobs: jobs)
+      XCTAssertFalse(driver.diagnosticEngine.hasErrors)
+    }
+  }
+
   func testExplicitModuleBuildEndToEnd() throws {
     try withTemporaryDirectory { path in
       try localFileSystem.changeCurrentWorkingDirectory(to: path)