Merge pull request #183 from uber/generator-collect_parsing_info-master

Expand tracking task ID to collecting more information

Generator/Package.swift

@@ -10,7 +10,7 @@ let package = Package(
     dependencies: [
         .package(url: "https://github.com/jpsim/SourceKitten.git", from: "0.20.0"),
         .package(url: "https://github.com/apple/swift-package-manager.git", from: "0.1.0"),
-        .package(url: "https://github.com/uber/swift-concurrency.git", from: "0.6.0"),
+        .package(url: "https://github.com/uber/swift-concurrency.git", from: "0.6.1"),
     ],
     targets: [
         .target(

Generator/Sources/NeedleFramework/Needle.swift

@@ -42,28 +42,29 @@ public class Needle {
     /// - parameter headerDocPath: The path to custom header doc file to be
     /// included at the top of the generated file.
     /// - parameter destinationPath: The path to export generated code to.
-    /// - parameter shouldTackTaskId: `true` if task IDs should be tracked
-    /// as tasks are executed. `false` otherwise. By tracking the task IDs,
-    /// if waiting on the completion of a task sequence times out, the
-    /// reported error contains the ID of the task that was being executed
-    /// when the timeout occurred. The tracking does incur a minor
-    /// performance cost. This value defaults to `false`.
-    public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldTackTaskId: Bool) {
+    /// - parameter shouldCollectParsingInfo: `true` if dependency graph
+    /// parsing information should be collected as tasks are executed. `false`
+    /// otherwise. By collecting execution information, if waiting on the
+    /// completion of a task sequence in the dependency parsing phase times out,
+    /// the reported error contains the relevant information when the timeout
+    /// occurred. The tracking does incur a minor performance cost. This value
+    /// defaults to `false`.
+    public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldCollectParsingInfo: Bool) {
         let sourceRootUrls = sourceRootPaths.map { (path: String) -> URL in
             URL(path: path)
         }
         #if DEBUG
-            let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId)
+            let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo)
         #else
-            let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId)
+            let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo)
         #endif
         let parser = DependencyGraphParser()
         do {
             let (components, imports) = try parser.parse(from: sourceRootUrls, withSourcesListFormat: sourcesListFormatValue, excludingFilesEndingWith: exclusionSuffixes, excludingFilesWithPaths: exclusionPaths, using: executor)
             let exporter = DependencyGraphExporter()
             try exporter.export(components, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath)
-        } catch DependencyGraphParserError.timeout(let sourcePath, let taskId) {
-            fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId).")
+        } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) {
+            fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit daemon process status: \(isSourceKitRunning).")
         } catch DependencyGraphExporterError.timeout(let componentName) {
             fatalError("Generating dependency provider for \(componentName) timed out.")
         } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) {

Generator/Sources/NeedleFramework/Parsing/DependencyGraphParser.swift

@@ -21,9 +21,10 @@ import Foundation
 /// Swift sources.
 enum DependencyGraphParserError: Error {
     /// Parsing a particular source file timed out. The associated values
-    /// are the path of the file being parsed and the ID of the task that
-    /// was being executed when the timeout occurred.
-    case timeout(String, Int)
+    /// are the path of the file being parsed, the ID of the task that
+    /// was being executed when the timeout occurred, and if the sourcekit
+    /// process was running when the timeout occurred.
+    case timeout(String, Int, Bool)
 }
 
 /// The entry utility for the parsing phase. The parser deeply scans a
@@ -119,7 +120,7 @@ class DependencyGraphParser {
                     imports.insert(statement)
                 }
             } catch SequenceExecutionError.awaitTimeout(let taskId) {
-                throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId)
+                throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId, isSourceKitRunning)
             } catch {
                 fatalError("Unhandled task execution error \(error)")
             }

Generator/Sources/NeedleFramework/Parsing/Pluginized/PluginizedDependencyGraphParser.swift

@@ -116,7 +116,7 @@ class PluginizedDependencyGraphParser {
                     imports.insert(statement)
                 }
             } catch SequenceExecutionError.awaitTimeout(let taskId) {
-                throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId)
+                throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId, isSourceKitRunning)
             } catch {
                 fatalError("Unhandled task execution error \(error)")
             }

Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift

@@ -39,28 +39,29 @@ public class PluginizedNeedle {
     /// - parameter headerDocPath: The path to custom header doc file to be
     /// included at the top of the generated file.
     /// - parameter destinationPath: The path to export generated code to.
-    /// - parameter shouldTackTaskId: `true` if task IDs should be tracked
-    /// as tasks are executed. `false` otherwise. By tracking the task IDs,
-    /// if waiting on the completion of a task sequence times out, the
-    /// reported error contains the ID of the task that was being executed
-    /// when the timeout occurred. The tracking does incur a minor
-    /// performance cost. This value defaults to `false`.
-    public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldTackTaskId: Bool) {
+    /// - parameter shouldCollectParsingInfo: `true` if dependency graph
+    /// parsing information should be collected as tasks are executed. `false`
+    /// otherwise. By collecting execution information, if waiting on the
+    /// completion of a task sequence in the dependency parsing phase times out,
+    /// the reported error contains the relevant information when the timeout
+    /// occurred. The tracking does incur a minor performance cost. This value
+    /// defaults to `false`.
+    public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldCollectParsingInfo: Bool) {
         let sourceRootUrls = sourceRootPaths.map { (path: String) -> URL in
             URL(path: path)
         }
         #if DEBUG
-            let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId)
+            let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo)
         #else
-            let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId)
+            let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo)
         #endif
         let parser = PluginizedDependencyGraphParser()
         do {
             let (components, pluginizedComponents, imports) = try parser.parse(from: sourceRootUrls, withSourcesListFormat: sourcesListFormatValue, excludingFilesEndingWith: exclusionSuffixes, excludingFilesWithPaths: exclusionPaths, using: executor)
             let exporter = PluginizedDependencyGraphExporter()
             try exporter.export(components, pluginizedComponents, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath)
-        } catch DependencyGraphParserError.timeout(let sourcePath, let taskId) {
-            fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId).")
+        } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) {
+            fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit daemon process status: \(isSourceKitRunning).")
         } catch DependencyGraphExporterError.timeout(let componentName) {
             fatalError("Generating dependency provider for \(componentName) timed out.")
         } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) {

Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift

@@ -0,0 +1,51 @@
+//
+//  Copyright (c) 2018. Uber Technologies
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+
+/// Check if the sourcekit daemon process is running by searching through
+/// all processes without ttys.
+var isSourceKitRunning: Bool {
+    // Select processes without controlling ttys in jobs format.
+    let result = ProcessUtilities.execute(process: "/bin/ps", withArguments: ["-xj"]).lowercased()
+    // These process names are found in library_wrapper_sourcekitd.swift
+    // of SourceKittenFramework.
+    return result.contains("libsourcekitdInProc") || result.contains("sourcekitd.framework")
+}
+
+/// A set of utility functions for running processes.
+class ProcessUtilities {
+
+    /// Execute the given process with given arguments and return the
+    /// standard output as a `String`.
+    ///
+    /// - parameter process: The process to run.
+    /// - parameter arguments: The list of arguments to supply to the
+    /// process.
+    /// - returns: The standard output content as a single `String`.
+    static func execute(process: String, withArguments arguments: [String] = []) -> String {
+        let task = Process()
+        task.launchPath = process
+        task.arguments = arguments
+        let pipe = Pipe()
+        task.standardOutput = pipe
+        task.launch()
+
+        let handle = pipe.fileHandleForReading
+        let data = handle.readDataToEndOfFile()
+        return String(data: data, encoding: .utf8) ?? ""
+    }
+}

Generator/Sources/needle/GenerateCommand.swift

@@ -46,7 +46,7 @@ class GenerateCommand: AbstractCommand {
         scanPlugins = parser.add(option: "--pluginized", shortName: "-p", kind: Bool.self, usage: "Whether or not to consider plugins when parsing.")
         additionalImports = parser.add(option: "--additional-imports", shortName: "-ai", kind: [String].self, usage: "Additional modules to import in the generated file, in addition to the ones parsed from source files.", completion: .none)
         headerDocPath = parser.add(option: "--header-doc", shortName: "-hd", kind: String.self, usage: "Path to custom header doc file to be included at the top of the generated file.", completion: .filename)
-        shouldTackTaskId = parser.add(option: "--track-task", shortName: "-tt", kind: Bool.self, usage: "Whether or not to track task IDs.")
+        shouldCollectParsingInfo = parser.add(option: "--collect-parsing-info", shortName: "-cpi", kind: Bool.self, usage: "Whether or not to collect information for parsing execution timeout errors.")
     }
 
     /// Execute the command.
@@ -64,11 +64,11 @@ class GenerateCommand: AbstractCommand {
                 let additionalImports = arguments.get(self.additionalImports) ?? []
                 let scanPlugins = arguments.get(self.scanPlugins) ?? false
                 let headerDocPath = arguments.get(self.headerDocPath) ?? nil
-                let shouldTackTaskId = arguments.get(self.shouldTackTaskId) ?? false
+                let shouldCollectParsingInfo = arguments.get(self.shouldCollectParsingInfo) ?? false
                 if scanPlugins {
-                    PluginizedNeedle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldTackTaskId: shouldTackTaskId)
+                    PluginizedNeedle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldCollectParsingInfo: shouldCollectParsingInfo)
                 } else {
-                    Needle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldTackTaskId: shouldTackTaskId)
+                    Needle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldCollectParsingInfo: shouldCollectParsingInfo)
                 }
             } else {
                 fatalError("Missing source files root directories.")
@@ -88,5 +88,5 @@ class GenerateCommand: AbstractCommand {
     private var additionalImports: OptionArgument<[String]>!
     private var scanPlugins: OptionArgument<Bool>!
     private var headerDocPath: OptionArgument<String>!
-    private var shouldTackTaskId: OptionArgument<Bool>!
+    private var shouldCollectParsingInfo: OptionArgument<Bool>!
 }