parallelize lute lint

[wmccrthy]

Distribute walking input paths and linting files across workers. Number of workers capped by the minimum of available system threads and # of lintee files.

Most of the changes are non-functional, just moving business logic into lintCore, so I didn't have to duplicate it and could load rules / config independently in created VMs due to constraints of what can be passed through. (side note: it'd be reallyyyy sweet if we had documentation on the vm lib and how to use it, as it took me a while to figure out what these constraints were and how to work around them. If i'm missing something, pls lmk 😄 )

Testing

All tests passing.

Also manually sanity checked on some local repos with the script in the collapsable below. Verified that every linted file had same the # of violations when linted with old / new build. Seeing pretty big improvements.
Comparing 4/24 nightly release (default) against release build from this branch (parallel)

Repo	Default	Parallel	Diff	Diff %	Old Violations	New Violations	Violation Diff
lua-apps	47.29s	8.69s	38.61s	81.6%	22821	22821	0
foundation	2.70s	0.57s	2.13s	78.9%	852	852	0
lute	0.56s	0.25s	0.31s	55.7%	20	20	0

Test script

local process = require("@std/process")
local path = require("@std/path")
local json = require("@std/json")
local types = require("./lute/cli/commands/lint/types")

local SEVERITY_MAP: { [number]: types.severity } = {
	[1] = "error",
	[2] = "warning",
	[3] = "info",
	[4] = "hint",
}

local function convertRange(range: any): { beginLine: number, beginColumn: number, endLine: number, endColumn: number }
	return {
		beginLine = range.start.line + 1,
		beginColumn = range.start.character + 1,
		endLine = range["end"].line + 1,
		endColumn = range["end"].character,
	}
end

local function convertDiagnostic(item: any, sourcepath: string): types.LintViolation
	local violation: types.LintViolation = {
		lintname = item.code,
		message = item.message,
		severity = (SEVERITY_MAP[item.severity] or "warning") :: types.severity,
		location = convertRange(item.range) :: any,
		sourcepath = path.parse(sourcepath),
		target = item.codeDescription,
	}

	if item.suggestedfix then
		violation.suggestedfix = {
			fix = item.suggestedfix.fix,
			location = if item.suggestedfix.range then convertRange(item.suggestedfix.range) :: any else nil,
		}
	end

	return violation
end

local function parseLintJsonOutput(jsonString: string): { [string]: { types.LintViolation } }
	local jsonStart = string.find(jsonString, "{")
	if not jsonStart then
		return {}
	end

	local parsed = json.deserialize(jsonString:sub(jsonStart, #jsonString)) :: any
	if not parsed or not parsed.items then
		return {}
	end

	local violationsByFile: { [string]: { types.LintViolation } } = {}

	for _, fileEntry in parsed.items do
		local filePath = fileEntry.uri :: string
		local violations: { types.LintViolation } = {}

		for _, item in fileEntry.items do
			table.insert(violations, convertDiagnostic(item, filePath))
		end

		violationsByFile[filePath] = violations
	end

	return violationsByFile
end

local function countViolations(results: { [string]: { types.LintViolation } }): number
	local count = 0
	for _, violations in results do
		count += #violations
	end
	return count
end

local function formatSeconds(value: number): string
	return string.format("%.2fs", value)
end

local function formatPercent(value: number): string
	return string.format("%.1f%%", value)
end

-- local build = path.join(".", "build", "xcode", "debug", "lute", "cli", "lute")

local releaseBuild = path.join(".", "build", "xcode", "release", "lute", "cli", "lute")

local matrix: { [string]: string } = {
	lute = "./",
	["lua-apps"] = "../roblox/lua-apps",
	["foundation"] = "../roblox/foundation",
}

type Result = {
	repo: string,
	old: number,
	new: number,
	diffSeconds: number,
	diffPercent: number,
	oldViolations: number,
	newViolations: number,
	violationDiff: number,
}

local results: { Result } = {}

for repo, repoPath in matrix do
	local start = os.clock()

	local normalLute = process.run({ "lute", "lint", "-j", repoPath })
	local normalTime = os.clock() - start

	start = os.clock()
	local parallelLint = process.run({ path.format(releaseBuild), "lint", "-j", repoPath })
	local parallelTime = os.clock() - start

	local specialResults = parseLintJsonOutput(parallelLint.stdout)

	local normalResults = parseLintJsonOutput(normalLute.stdout)

	for file, violations in specialResults do
		local normalViolations = normalResults[file]
		assert(normalViolations ~= nil, `Missing normal lint results for {file}`)
		assert(#violations == #normalViolations, `Violation count mismatch for {file}`)
	end

	for file, violations in normalResults do
		local specialViolations = specialResults[file]
		assert(specialViolations ~= nil, `Missing parallel lint results for {file}`)
		assert(#violations == #specialViolations, `Violation count mismatch for {file}`)
	end

	local normalViolationCount = countViolations(normalResults)
	local parallelViolationCount = countViolations(specialResults)
	local diffSeconds = normalTime - parallelTime
	local diffPercent = if normalTime > 0 then (diffSeconds / normalTime) * 100 else 0

	table.insert(results, {
		repo = repo,
		old = normalTime,
		new = parallelTime,
		diffSeconds = diffSeconds,
		diffPercent = diffPercent,
		oldViolations = normalViolationCount,
		newViolations = parallelViolationCount,
		violationDiff = parallelViolationCount - normalViolationCount,
	})
end

-- pretty print results in markdown table
print("| Repo | Old | New | Diff | Diff % | Old Violations | New Violations | Violation Diff |")
print("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")

for _, result in results do
	print(
		`| {result.repo} | {formatSeconds(result.old)} | {formatSeconds(result.new)} | {formatSeconds(
			result.diffSeconds
		)} | {formatPercent(result.diffPercent)} | {result.oldViolations} | {result.newViolations} | {result.violationDiff} |`
	)
end

[skberkeley]

This is a pretty large change and difficult to review at the moment. It seems like this is currently a refactor combined with the functional changes to parallelize linting. If that's the case, could I ask that you split out the refactor part into a separate PR to make it easier to review?

[wmccrthy]

This is a pretty large change and difficult to review at the moment. It seems like this is currently a refactor combined with the functional changes to parallelize linting. If that's the case, could I ask that you split out the refactor part into a separate PR to make it easier to review?

Sure! Some of the refactor is tightly coupled to the constraints of using vm, so even with an additional PR I will have to move a couple things around from the refactor PR -> parallel PR, but it should be more digestible regardless

[wmccrthy]

This is a pretty large change and difficult to review at the moment. It seems like this is currently a refactor combined with the functional changes to parallelize linting. If that's the case, could I ask that you split out the refactor part into a separate PR to make it easier to review?

@skberkeley refactor up here, i'll rebase and clean up this branch once that is merged