From 0f6bcde461b9d59ba2f83a2414f1e85e2af609c2 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 2 Nov 2018 18:27:44 -0700 Subject: [PATCH 1/3] Expand tracking task ID to collecting more information `--track-id` is expanded to `--collect-parsing-info` to include more information. This changeset adds the information of if SourceKit deamon process is running at the time the dependency graph parsing phase times out. --- .../Sources/NeedleFramework/Needle.swift | 23 ++++----- .../Parsing/DependencyGraphParser.swift | 9 ++-- .../PluginizedDependencyGraphParser.swift | 2 +- .../Pluginized/PluginizedNeedle.swift | 23 ++++----- .../Utilities/ProcessUtilities.swift | 49 +++++++++++++++++++ .../Sources/needle/GenerateCommand.swift | 10 ++-- 6 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift diff --git a/Generator/Sources/NeedleFramework/Needle.swift b/Generator/Sources/NeedleFramework/Needle.swift index f7a87595..12e784cc 100644 --- a/Generator/Sources/NeedleFramework/Needle.swift +++ b/Generator/Sources/NeedleFramework/Needle.swift @@ -42,28 +42,29 @@ public class Needle { /// - parameter headerDocPath: The path to custom header doc file to be /// included at the top of the generated file. /// - parameter destinationPath: The path to export generated code to. - /// - parameter shouldTackTaskId: `true` if task IDs should be tracked - /// as tasks are executed. `false` otherwise. By tracking the task IDs, - /// if waiting on the completion of a task sequence times out, the - /// reported error contains the ID of the task that was being executed - /// when the timeout occurred. The tracking does incur a minor - /// performance cost. This value defaults to `false`. - public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldTackTaskId: Bool) { + /// - parameter shouldCollectParsingInfo: `true` if dependency graph + /// parsing information should be collected as tasks are executed. `false` + /// otherwise. By collecting execution information, if waiting on the + /// completion of a task sequence in the dependency parsing phase times out, + /// the reported error contains the relevant information when the timeout + /// occurred. The tracking does incur a minor performance cost. This value + /// defaults to `false`. + public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldCollectParsingInfo: Bool) { let sourceRootUrls = sourceRootPaths.map { (path: String) -> URL in URL(path: path) } #if DEBUG - let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId) + let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) #else - let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId) + let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) #endif let parser = DependencyGraphParser() do { let (components, imports) = try parser.parse(from: sourceRootUrls, withSourcesListFormat: sourcesListFormatValue, excludingFilesEndingWith: exclusionSuffixes, excludingFilesWithPaths: exclusionPaths, using: executor) let exporter = DependencyGraphExporter() try exporter.export(components, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath) - } catch DependencyGraphParserError.timeout(let sourcePath, let taskId) { - fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId).") + } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) { + fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit deamon process status: \(isSourceKitRunning).") } catch DependencyGraphExporterError.timeout(let componentName) { fatalError("Generating dependency provider for \(componentName) timed out.") } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) { diff --git a/Generator/Sources/NeedleFramework/Parsing/DependencyGraphParser.swift b/Generator/Sources/NeedleFramework/Parsing/DependencyGraphParser.swift index c71cec91..9b680739 100644 --- a/Generator/Sources/NeedleFramework/Parsing/DependencyGraphParser.swift +++ b/Generator/Sources/NeedleFramework/Parsing/DependencyGraphParser.swift @@ -21,9 +21,10 @@ import Foundation /// Swift sources. enum DependencyGraphParserError: Error { /// Parsing a particular source file timed out. The associated values - /// are the path of the file being parsed and the ID of the task that - /// was being executed when the timeout occurred. - case timeout(String, Int) + /// are the path of the file being parsed, the ID of the task that + /// was being executed when the timeout occurred, and if the sourcekit + /// process was running when the timeout occurred. + case timeout(String, Int, Bool) } /// The entry utility for the parsing phase. The parser deeply scans a @@ -119,7 +120,7 @@ class DependencyGraphParser { imports.insert(statement) } } catch SequenceExecutionError.awaitTimeout(let taskId) { - throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId) + throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId, isSourceKitRunning) } catch { fatalError("Unhandled task execution error \(error)") } diff --git a/Generator/Sources/NeedleFramework/Parsing/Pluginized/PluginizedDependencyGraphParser.swift b/Generator/Sources/NeedleFramework/Parsing/Pluginized/PluginizedDependencyGraphParser.swift index a4c07f90..7835c3e8 100644 --- a/Generator/Sources/NeedleFramework/Parsing/Pluginized/PluginizedDependencyGraphParser.swift +++ b/Generator/Sources/NeedleFramework/Parsing/Pluginized/PluginizedDependencyGraphParser.swift @@ -116,7 +116,7 @@ class PluginizedDependencyGraphParser { imports.insert(statement) } } catch SequenceExecutionError.awaitTimeout(let taskId) { - throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId) + throw DependencyGraphParserError.timeout(urlHandle.fileUrl.absoluteString, taskId, isSourceKitRunning) } catch { fatalError("Unhandled task execution error \(error)") } diff --git a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift index 403205a8..548e2473 100644 --- a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift +++ b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift @@ -39,28 +39,29 @@ public class PluginizedNeedle { /// - parameter headerDocPath: The path to custom header doc file to be /// included at the top of the generated file. /// - parameter destinationPath: The path to export generated code to. - /// - parameter shouldTackTaskId: `true` if task IDs should be tracked - /// as tasks are executed. `false` otherwise. By tracking the task IDs, - /// if waiting on the completion of a task sequence times out, the - /// reported error contains the ID of the task that was being executed - /// when the timeout occurred. The tracking does incur a minor - /// performance cost. This value defaults to `false`. - public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldTackTaskId: Bool) { + /// - parameter shouldCollectParsingInfo: `true` if dependency graph + /// parsing information should be collected as tasks are executed. `false` + /// otherwise. By collecting execution information, if waiting on the + /// completion of a task sequence in the dependency parsing phase times out, + /// the reported error contains the relevant information when the timeout + /// occurred. The tracking does incur a minor performance cost. This value + /// defaults to `false`. + public static func generate(from sourceRootPaths: [String], withSourcesListFormat sourcesListFormatValue: String? = nil, excludingFilesEndingWith exclusionSuffixes: [String], excludingFilesWithPaths exclusionPaths: [String], with additionalImports: [String], _ headerDocPath: String?, to destinationPath: String, shouldCollectParsingInfo: Bool) { let sourceRootUrls = sourceRootPaths.map { (path: String) -> URL in URL(path: path) } #if DEBUG - let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId) + let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) #else - let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldTackTaskId) + let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) #endif let parser = PluginizedDependencyGraphParser() do { let (components, pluginizedComponents, imports) = try parser.parse(from: sourceRootUrls, withSourcesListFormat: sourcesListFormatValue, excludingFilesEndingWith: exclusionSuffixes, excludingFilesWithPaths: exclusionPaths, using: executor) let exporter = PluginizedDependencyGraphExporter() try exporter.export(components, pluginizedComponents, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath) - } catch DependencyGraphParserError.timeout(let sourcePath, let taskId) { - fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId).") + } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) { + fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit deamon process status: \(isSourceKitRunning).") } catch DependencyGraphExporterError.timeout(let componentName) { fatalError("Generating dependency provider for \(componentName) timed out.") } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) { diff --git a/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift b/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift new file mode 100644 index 00000000..fc001411 --- /dev/null +++ b/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift @@ -0,0 +1,49 @@ +// +// Copyright (c) 2018. Uber Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation + +/// Check if the sourcekit deamon process is running by searching through +/// all processes without ttys. +var isSourceKitRunning: Bool { + // Select processes without controlling ttys in jobs format. + let result = ProcessUtilities.execute(process: "/bin/ps", withArguments: ["-xj"]) + return result.lowercased().contains("sourcekit") +} + +/// A set of utility functions for running processes. +class ProcessUtilities { + + /// Execute the given process with given arguments and return the + /// standard output as a `String`. + /// + /// - parameter process: The process to run. + /// - parameter arguments: The list of arguments to supply to the + /// process. + /// - returns: The standard output content as a single `String`. + static func execute(process: String, withArguments arguments: [String] = []) -> String { + let task = Process() + task.launchPath = process + task.arguments = arguments + let pipe = Pipe() + task.standardOutput = pipe + task.launch() + + let handle = pipe.fileHandleForReading + let data = handle.readDataToEndOfFile() + return String(data: data, encoding: .utf8) ?? "" + } +} diff --git a/Generator/Sources/needle/GenerateCommand.swift b/Generator/Sources/needle/GenerateCommand.swift index 9c518dfa..9c1e3b75 100644 --- a/Generator/Sources/needle/GenerateCommand.swift +++ b/Generator/Sources/needle/GenerateCommand.swift @@ -46,7 +46,7 @@ class GenerateCommand: AbstractCommand { scanPlugins = parser.add(option: "--pluginized", shortName: "-p", kind: Bool.self, usage: "Whether or not to consider plugins when parsing.") additionalImports = parser.add(option: "--additional-imports", shortName: "-ai", kind: [String].self, usage: "Additional modules to import in the generated file, in addition to the ones parsed from source files.", completion: .none) headerDocPath = parser.add(option: "--header-doc", shortName: "-hd", kind: String.self, usage: "Path to custom header doc file to be included at the top of the generated file.", completion: .filename) - shouldTackTaskId = parser.add(option: "--track-task", shortName: "-tt", kind: Bool.self, usage: "Whether or not to track task IDs.") + shouldCollectParsingInfo = parser.add(option: "--collect-parsing-info", shortName: "-cpi", kind: Bool.self, usage: "Whether or not to collect information for parsing execution timeout errors.") } /// Execute the command. @@ -64,11 +64,11 @@ class GenerateCommand: AbstractCommand { let additionalImports = arguments.get(self.additionalImports) ?? [] let scanPlugins = arguments.get(self.scanPlugins) ?? false let headerDocPath = arguments.get(self.headerDocPath) ?? nil - let shouldTackTaskId = arguments.get(self.shouldTackTaskId) ?? false + let shouldCollectParsingInfo = arguments.get(self.shouldCollectParsingInfo) ?? false if scanPlugins { - PluginizedNeedle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldTackTaskId: shouldTackTaskId) + PluginizedNeedle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldCollectParsingInfo: shouldCollectParsingInfo) } else { - Needle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldTackTaskId: shouldTackTaskId) + Needle.generate(from: sourceRootPaths, withSourcesListFormat: sourcesListFormat, excludingFilesEndingWith: excludeSuffixes, excludingFilesWithPaths: excludePaths, with: additionalImports, headerDocPath, to: destinationPath, shouldCollectParsingInfo: shouldCollectParsingInfo) } } else { fatalError("Missing source files root directories.") @@ -88,5 +88,5 @@ class GenerateCommand: AbstractCommand { private var additionalImports: OptionArgument<[String]>! private var scanPlugins: OptionArgument! private var headerDocPath: OptionArgument! - private var shouldTackTaskId: OptionArgument! + private var shouldCollectParsingInfo: OptionArgument! } From 173b8975b8e9ef3b54a409d90aca7d54d72f45a5 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Mon, 5 Nov 2018 11:00:26 -0800 Subject: [PATCH 2/3] Updating to swift-concurrency 0.6.1 --- Generator/Package.swift | 2 +- Generator/Sources/NeedleFramework/Needle.swift | 4 ++-- .../Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Generator/Package.swift b/Generator/Package.swift index 14ce260f..11d06884 100644 --- a/Generator/Package.swift +++ b/Generator/Package.swift @@ -10,7 +10,7 @@ let package = Package( dependencies: [ .package(url: "https://github.com/jpsim/SourceKitten.git", from: "0.20.0"), .package(url: "https://github.com/apple/swift-package-manager.git", from: "0.1.0"), - .package(url: "https://github.com/uber/swift-concurrency.git", from: "0.6.0"), + .package(url: "https://github.com/uber/swift-concurrency.git", from: "0.6.1"), ], targets: [ .target( diff --git a/Generator/Sources/NeedleFramework/Needle.swift b/Generator/Sources/NeedleFramework/Needle.swift index 12e784cc..9ee9839b 100644 --- a/Generator/Sources/NeedleFramework/Needle.swift +++ b/Generator/Sources/NeedleFramework/Needle.swift @@ -54,9 +54,9 @@ public class Needle { URL(path: path) } #if DEBUG - let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) + let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo) #else - let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) + let executor = ConcurrentSequenceExecutor(name: "Needle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo) #endif let parser = DependencyGraphParser() do { diff --git a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift index 548e2473..0d6b1422 100644 --- a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift +++ b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift @@ -51,9 +51,9 @@ public class PluginizedNeedle { URL(path: path) } #if DEBUG - let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) + let executor: SequenceExecutor = ProcessInfo().environment["SINGLE_THREADED"] != nil ? SerialSequenceExecutor() : ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo) #else - let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTackTaskId: shouldCollectParsingInfo) + let executor = ConcurrentSequenceExecutor(name: "PluginizedNeedle.generate", qos: .userInteractive, shouldTrackTaskId: shouldCollectParsingInfo) #endif let parser = PluginizedDependencyGraphParser() do { From 083be8f28d14d026ead23d8c8830cc4d2aa7ab7a Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Mon, 5 Nov 2018 13:27:42 -0800 Subject: [PATCH 3/3] Use SourceKittenFramework's service names for SourceKit process searching Also fixed some spelling errors. --- Generator/Sources/NeedleFramework/Needle.swift | 2 +- .../NeedleFramework/Pluginized/PluginizedNeedle.swift | 2 +- .../NeedleFramework/Utilities/ProcessUtilities.swift | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Generator/Sources/NeedleFramework/Needle.swift b/Generator/Sources/NeedleFramework/Needle.swift index 9ee9839b..aadbbdc1 100644 --- a/Generator/Sources/NeedleFramework/Needle.swift +++ b/Generator/Sources/NeedleFramework/Needle.swift @@ -64,7 +64,7 @@ public class Needle { let exporter = DependencyGraphExporter() try exporter.export(components, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath) } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) { - fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit deamon process status: \(isSourceKitRunning).") + fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit daemon process status: \(isSourceKitRunning).") } catch DependencyGraphExporterError.timeout(let componentName) { fatalError("Generating dependency provider for \(componentName) timed out.") } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) { diff --git a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift index 0d6b1422..dcf19fec 100644 --- a/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift +++ b/Generator/Sources/NeedleFramework/Pluginized/PluginizedNeedle.swift @@ -61,7 +61,7 @@ public class PluginizedNeedle { let exporter = PluginizedDependencyGraphExporter() try exporter.export(components, pluginizedComponents, with: imports + additionalImports, to: destinationPath, using: executor, include: headerDocPath) } catch DependencyGraphParserError.timeout(let sourcePath, let taskId, let isSourceKitRunning) { - fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit deamon process status: \(isSourceKitRunning).") + fatalError("Parsing Swift source file at \(sourcePath) timed out when executing task with ID \(taskId). SourceKit daemon process status: \(isSourceKitRunning).") } catch DependencyGraphExporterError.timeout(let componentName) { fatalError("Generating dependency provider for \(componentName) timed out.") } catch DependencyGraphExporterError.unableToWriteFile(let outputFile) { diff --git a/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift b/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift index fc001411..aedcb386 100644 --- a/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift +++ b/Generator/Sources/NeedleFramework/Utilities/ProcessUtilities.swift @@ -16,12 +16,14 @@ import Foundation -/// Check if the sourcekit deamon process is running by searching through +/// Check if the sourcekit daemon process is running by searching through /// all processes without ttys. var isSourceKitRunning: Bool { // Select processes without controlling ttys in jobs format. - let result = ProcessUtilities.execute(process: "/bin/ps", withArguments: ["-xj"]) - return result.lowercased().contains("sourcekit") + let result = ProcessUtilities.execute(process: "/bin/ps", withArguments: ["-xj"]).lowercased() + // These process names are found in library_wrapper_sourcekitd.swift + // of SourceKittenFramework. + return result.contains("libsourcekitdInProc") || result.contains("sourcekitd.framework") } /// A set of utility functions for running processes.