SwiftPM 6.4 support - Swift Build support

Author: johnfairh

CHANGELOG.md

@@ -9,6 +9,9 @@
 * New syntax, attribute and declaration kinds introduced in Swift 6.4.  
   [John Fairhurst](https://github.com/johnfairh)
 
+* Docs generation and code completion with Swift PM 6.4.  
+  [John Fairhurst](https://github.com/johnfairh)
+
 #### Bug Fixes
 
 * None.

Source/SourceKittenFramework/Module.swift

@@ -29,24 +29,34 @@ public struct Module {
     /**
      Failable initializer to create a Module from a Swift Package Manager build record.
 
-     Use this initializer when the package has already been built and the `.build` directory exists.
+     Use this initializer when the package has already been built and the `.build` directory exists
+     and the build system is not Swift Build.
+
+     This initializer does not work with the Swift Build backend that is the default in Swift PM 6.4.
+     Use `init(spmArguments:spmName:inPath:)` instead, which will build the package.
 
      - parameter spmName: Module name.  Will use some non-Test module that is part of the
                           package if `nil`.
      - parameter path:    Path of the directory containing the SPM `.build` directory.
                           Uses the current directory by default.
      */
+    @available(*, deprecated, message: """
+                Use init(spmArguments:spmName:inPath:) instead.
+                This initializer does not support the Swift Build backend that is the default for SwiftPM 6.4
+                """)
     public init?(spmName: String? = nil, inPath path: String = FileManager.default.currentDirectoryPath) {
         guard let results = SwiftPM.fromDebugYaml(moduleName: spmName, inPath: path) else {
             return nil
         }
+        fputs("Using module data from debug.yaml\n", stderr)
         self.init(name: results.0, compilerArguments: results.1)
     }
 
     /**
      Failable initializer to create a Module by building a Swift Package Manager project.
 
-     Use this initializer if the package has not been built or may have changed since last built.
+     As of SwiftPM 6.4 this is likely to build the package even it has been previously built because
+     of the build system changing to Swift Build.
 
      - parameter spmArguments: Additional arguments to pass to `swift build`
      - parameter spmName: Module name.  Will use some non-Test module that is part of the
@@ -55,12 +65,64 @@ public struct Module {
                           Uses the current directory by default.
      */
     public init?(spmArguments: [String], spmName: String? = nil, inPath path: String = FileManager.default.currentDirectoryPath) {
+        /*
+         1. `build -v` to get a build directory and maybe some compile commands for (3).
+             Fast if already built.
+         2. If there is a `debug.yaml` file then use it -- pre-6.4 / swiftbuild back-end.  Done.
+         3. Check for compiler flags in the build log from (1) -- if successful then Done.
+         4. `package clean` and `build -v` --- hope that (3) failed because module already built
+            before (1) so it did not rebuild anything.
+         5. Check for compiler flags in the build log from (4) -- if successful then Done.
+         6. Fail.
+
+         The swiftbuild backend uses a proprietary binary-ish msgpack format instead of the debug.yaml.
+         We could decode it and save the extra clean-build-verbose step but would mean heavier
+         dependencies and more fragility.
+
+         The swiftbuild backend produces an XCBuildData/manifest.json that looks promising but does not
+         contain compiler arguments.
+        */
+
+        // 1. Initial build check
         fputs("Running swift build\n", stderr)
-        guard SwiftPM.runBuild(arguments: spmArguments, inPath: path) != nil else {
+        guard let buildResults = SwiftPM.runVerboseBuild(arguments: spmArguments, inPath: path) else {
+            return nil
+        }
+
+        // 2. Pre-Swift Build solution
+        if SwiftPM.hasDebugYaml(inPath: path) {
+            if let info = SwiftPM.fromDebugYaml(moduleName: spmName, inPath: path) {
+                fputs("Using module data from debug.yaml\n", stderr)
+                self.init(name: info.0, compilerArguments: info.1)
+                return
+            }
+            return nil
+        }
+
+        // 3. See if the (1) build built our module
+        if let info = SwiftPM.fromBuildResults(buildResults, moduleName: spmName) {
+            fputs("Using module data from build output\n", stderr)
+            self.init(name: info.0, compilerArguments: info.1)
+            return
+        }
+
+        // 4. Clean & Build
+        fputs("Running swift package clean and build\n", stderr)
+        guard SwiftPM.runClean(inPath: path) != nil,
+              let secondBuildResults = SwiftPM.runVerboseBuild(arguments: spmArguments, inPath: path) else {
             return nil
         }
 
-        self.init(spmName: spmName, inPath: path)
+        // 5. Should have our module now
+        if let info = SwiftPM.fromBuildResults(secondBuildResults, moduleName: spmName) {
+            fputs("Using module data from clean build output\n", stderr)
+            self.init(name: info.0, compilerArguments: info.1)
+            return
+        }
+
+        let path = secondBuildResults.save(prefix: "swift-build")
+        fputs("Could not parse module name '\(spmName ?? "(any)")' from swift build output: \(path)\n", stderr)
+        return nil
     }
 
     /**

Source/SourceKittenFramework/SwiftPM.swift

@@ -11,6 +11,11 @@ enum SwiftPM {
         return URL(fileURLWithPath: path).appendingPathComponent(".build/debug.yaml").path
     }
 
+    /// Is there a `debug.yaml` file in the .bulid directory
+    static func hasDebugYaml(inPath path: String) -> Bool {
+        return FileManager.default.fileExists(atPath: debugYamlPath(inPath: path))
+    }
+
     /**
      Get module build flags and optionally name from the `debug.yaml` file produced by the default SwiftPM build
      system earlier than Swift 6.4.
@@ -55,7 +60,7 @@ enum SwiftPM {
     // MARK: Build commands
 
     /**
-     Run  a build command.  If it works then return the results otherwise report the error and save the output.
+     Run a build command.  If it works then return the results otherwise report the error and save the output.
 
      - parameter arguments: Program name and arguments
      - parameter path: Working directory for program
@@ -75,6 +80,37 @@ enum SwiftPM {
     static func runBuild(arguments: [String], inPath path: String) -> Exec.Results? {
         return runCheckedCommand(arguments: ["swift", "build"] + arguments, inPath: path, msgName: "swift build")
     }
+
+    /// Run `swift build -v`. Return the successful results or `nil` and report the error.
+    static func runVerboseBuild(arguments: [String], inPath path: String) -> Exec.Results? {
+        return runBuild(arguments: ["-v"] + arguments, inPath: path)
+    }
+
+    /// Run `swift package clean`. Return the successful results or `nil` and report the error.
+    static func runClean(inPath path: String) -> Exec.Results? {
+        return runCheckedCommand(arguments: ["swift", "package", "clean"], inPath: path, msgName: "swift package clean")
+    }
+
+    // MARK: Build output parsing
+
+    /**
+     Given some results from `swift build -v`, find the compiler arguments for the module.
+
+     - parameter swiftBuildOutput: Output of `swift build -v
+     - parameter moduleName: Module of interest, `nil` to use any
+     - returns: tuple of module name and compiler args, or `nil` on failure
+     */
+    static func fromBuildResults(_ results: Exec.Results, moduleName: String?) -> (String, [String])? {
+        guard let buildOutput = results.string,
+              let swiftArgs = SourceKittenFramework.parseCompilerArguments(
+                xcodebuildOutput: buildOutput,
+                language: .swift,
+                moduleName: moduleName),
+              let actualModuleName = SourceKittenFramework.moduleName(fromArguments: swiftArgs) else {
+            return nil
+        }
+        return (actualModuleName, swiftArgs)
+    }
 }
 
 // MARK: Yams.Node helpers for SwiftPM's debug.yaml

Source/SourceKittenFramework/Xcode.swift

@@ -119,7 +119,8 @@ func filterForSourceKit(arguments args: [String]) -> [String] {
             "-use-frontend-parseable-output",
             "-incremental",
             "-serialize-diagnostics",
-            "-emit-dependencies"
+            "-emit-dependencies",
+            "-v"
         ].contains($0)
     }.map {
         if $0 == "-O" {

Source/sourcekitten/Complete.swift

@@ -43,7 +43,7 @@ extension SourceKitten {
                     args.append(contentsOf: ["-sdk", sdkPath()])
                 }
             } else {
-                guard let module = Module(spmName: spmModule) else {
+                guard let module = Module(spmArguments: [], spmName: spmModule) else {
                     throw SourceKittenError.invalidArgument(description: "Bad module name")
                 }
                 args = module.compilerArguments