editor_helper.lua

--DONT_ANALYZE
--[[HOTRELOAD
	run_test("test/tests/editor_helper.lua")
	--os.execute("luajit nattlua.lua build fast && luajit nattlua.lua install")
]]
local pairs = _G.pairs
local ipairs = _G.ipairs
local type = _G.type
local table = _G.table
local setmetatable = _G.setmetatable
local pcall = _G.pcall
local debug = _G.debug
local assert = _G.assert
local Compiler = require("nattlua.compiler").New
local formating = require("nattlua.other.formating")
local Union = require("nattlua.types.union").Union
local Table = require("nattlua.types.table").Table
local runtime_syntax = require("nattlua.syntax.runtime")
local typesystem_syntax = require("nattlua.syntax.typesystem")
local callstack = require("nattlua.other.callstack")
local bit = require("nattlua.other.bit")
local class = require("nattlua.other.class")
local fs = require("nattlua.other.fs")
local Code = require("nattlua.code").New
local Emitter = require("nattlua.emitter.emitter").New
local BuildBaseEnvironment = require("nattlua.base_environment").BuildBaseEnvironment
local runtime_env, typesystem_env, base_node_to_type = BuildBaseEnvironment()
local path_util = require("nattlua.other.path")
local Token = require("nattlua.lexer.token")
local META = class.CreateTemplate("editor_helper")
META:GetSet("WorkingDirectory", "./")
local format_emitter_defaults = {
	pretty_print = true,
	string_quote = "\"",
	no_semicolon = true,
	comment_type_annotations = true,
	type_annotations = "explicit",
	force_parenthesis = true,
	omit_parentheses_for_single_table_call = true,
	trailing_newline = true,
}

local function apply_format_emitter_defaults(emitter_config)
	for k, v in pairs(format_emitter_defaults) do
		if emitter_config[k] == nil then emitter_config[k] = v end
	end
end

function META:SetWorkingDirectory(dir)
	self.WorkingDirectory = dir
end

META:GetSet("ConfigFunction", function()
	return
end)

function META:GetProjectConfig(what, path)
	local get_config = self.ConfigFunction
	local config = get_config(path)

	if config then
		local entry = config[what]

		if not entry and config.commands then entry = config.commands[what] end

		local sub_config

		if type(entry) == "function" then
			sub_config = entry()
		elseif type(entry) == "table" then
			if type(entry.cb) == "function" then
				sub_config = entry.cb()
			else
				sub_config = entry
			end
		end

		if sub_config then
			if sub_config.ignorefiles == nil and config.ignorefiles ~= nil then
				sub_config.ignorefiles = config.ignorefiles
			end

			sub_config.root_directory = config.config_dir
			return sub_config
		end
	end
end

function META.New()
	local node_to_type = {}

	for k, v in pairs(base_node_to_type) do
		node_to_type[k] = v
	end

	return META.NewObject(
		{
			TempFiles = {},
			LoadedFiles = {},
			DirtyFiles = {},
			DirtySince = {},
			OpenFiles = {},
			VisibleFiles = {},
			PublishedDiagnostics = {},
			UseVisibleFilesForOpen = false,
			InteractiveRefreshDelay = 0.25,
			OnYield = false,
			debug = false,
			node_to_type = node_to_type,
		},
		true
	)
end

function META:NodeToType(obj)
	local node = self.node_to_type[obj]

	if node and node.GetStartStop then return node end

	return nil
end

function META:GetCompilerConfig(path)
	local project_config = self.ConfigFunction(path) or {}
	local cfg = self:GetProjectConfig("get-compiler-config", path) or {}
	cfg.lexer = cfg.lexer or {}
	cfg.emitter = cfg.emitter or {}
	cfg.analyzer = cfg.analyzer or {}
	cfg.lsp = cfg.lsp or {}
	cfg.parser = cfg.parser or {}
	cfg.root_directory = cfg.root_directory or project_config.config_dir or self:GetWorkingDirectory()

	if
		cfg.root_directory and
		cfg.root_directory ~= "" and
		cfg.root_directory:sub(-1) ~= "/"
	then
		cfg.root_directory = cfg.root_directory .. "/"
	end

	cfg.parser.root_directory = cfg.parser.root_directory or cfg.root_directory
	cfg.analyzer.root_directory = cfg.analyzer.root_directory or cfg.root_directory

	if self.workspace_config then
		if self.workspace_config.removeUnusedOnSave then
			cfg.emitter.remove_unused = true
			cfg.analyzer.remove_unused = true
		end
	end

	if cfg.emitter.type_annotations == nil then
		cfg.emitter.type_annotations = true
	end

	if cfg.analyzer.should_crawl_untyped_functions == nil then
		cfg.analyzer.should_crawl_untyped_functions = true
	end

	local function wrap_check_timeout(stage_cfg)
		if stage_cfg.__nattlua_lsp_check_timeout_wrapped then return end

		local previous = stage_cfg.check_timeout
		stage_cfg.check_timeout = function(...)
			if previous then previous(...) end

			self:Yield()
		end
		stage_cfg.__nattlua_lsp_check_timeout_wrapped = true
	end

	wrap_check_timeout(cfg.lexer)
	wrap_check_timeout(cfg.parser)
	wrap_check_timeout(cfg.analyzer)
	return cfg
end

function META:IsLightLspMode(path)
	local cfg = self:GetCompilerConfig(path)
	return cfg.lsp and (cfg.lsp.analyze == false or cfg.lsp.parse_only) or false
end

function META:DebugLog(str)
	if self.debug then print(coroutine.running(), str) end
end

function META:Yield()
	if self.OnYield then return self:OnYield() end
end

function META:GetTime()
	if self.Now then return self:Now() end

	return os.clock()
end

do
	function META:IsLoaded(path)
		path = path_util.Normalize(path)
		return self.LoadedFiles[path] ~= nil
	end

	function META:IsDirty(path)
		path = path_util.Normalize(path)
		return self.DirtyFiles[path] == true
	end

	function META:MarkDirty(path)
		path = path_util.Normalize(path)
		self.DirtyFiles[path] = true
		self.DirtySince[path] = self:GetTime()
	end

	function META:ClearDirty(path)
		path = path_util.Normalize(path)
		self.DirtyFiles[path] = nil
		self.DirtySince[path] = nil
	end

	function META:ShouldDeferInteractiveRefresh(path)
		path = path_util.Normalize(path)

		if not self:IsDirty(path) then return false end

		local dirty_since = self.DirtySince[path]

		if not dirty_since then return false end

		return self:GetTime() - dirty_since < (self.InteractiveRefreshDelay or 0)
	end

	function META:GetFile(path)
		path = path_util.Normalize(path)

		if not self.LoadedFiles[path] then
			self:DebugLog("[ " .. path .. " ] is not loaded")
			self:DebugLog("=== these are loaded ===")

			for k, v in pairs(self.LoadedFiles) do
				self:DebugLog("[ " .. k .. " ] is loaded")
			end

			self:DebugLog("===")
			error(path .. " not loaded", 2)
		end

		return self.LoadedFiles[path]
	end

	function META:IsParsed(path)
		path = path_util.Normalize(path)
		local data = self.LoadedFiles[path]
		return data ~= nil and data.parsed == true
	end

	function META:IsAnalyzed(path)
		path = path_util.Normalize(path)
		local data = self.LoadedFiles[path]
		return data ~= nil and data.analyzed == true
	end

	function META:LoadFile(path, code, tokens, compiler, analyzed)
		path = path_util.Normalize(path)
		self.LoadedFiles[path] = {
			code = code,
			tokens = tokens,
			compiler = compiler,
			parsed = true,
			analyzed = analyzed == true,
		}
	end

	function META:MarkAnalyzed(path)
		path = path_util.Normalize(path)
		local data = self.LoadedFiles[path]

		if data then data.analyzed = true end
	end

	function META:UnloadFile(path)
		path = path_util.Normalize(path)
		self.LoadedFiles[path] = nil
	end
end

do
	function META:SetFileContent(path, code)
		path = path_util.Normalize(path)
		self.TempFiles[path] = code
	end

	function META:GetFileContent(path)
		path = path_util.Normalize(path)

		if not self.TempFiles[path] then
			self:DebugLog("[ " .. path .. " ] content is not loaded")
			self:DebugLog("=== these are loaded ===")

			for k, v in pairs(self.TempFiles) do
				self:DebugLog("[ " .. k .. " ] content is loaded")
			end

			self:DebugLog("===")
			error(path .. " is not loaded", 2)
		end

		return self.TempFiles[path]
	end

	function META:GetCurrentContent(path)
		path = path_util.Normalize(path)

		if self.TempFiles[path] ~= nil then return self.TempFiles[path] end

		return fs.read(path)
	end
end

local function normalize_path(path)
	if not path then return nil end

	if path:sub(1, 1) == "@" then path = path:sub(2) end

	return path_util.Normalize(path)
end

local function has_force_analyze_directive(content)
	return content and
		(
			content:match("^%s*%-%-%s*ANALYZE") or
			content:match("^%#%!.-%s*%-%-%s*ANALYZE")
		)
end

function META:SetVisibleFiles(paths)
	local next_visible = {}
	local newly_visible = {}

	for _, path in ipairs(paths or {}) do
		path = normalize_path(path)

		if path then
			next_visible[path] = true

			if not self.VisibleFiles[path] then table.insert(newly_visible, path) end
		end
	end

	self.VisibleFiles = next_visible
	self.UseVisibleFilesForOpen = true
	return newly_visible
end

function META:IsVisibleFile(path)
	path = normalize_path(path)
	return path ~= nil and self.VisibleFiles[path] or false
end

function META:ShouldRecompileOnOpen(path)
	if not self.UseVisibleFilesForOpen then return true end

	return self:IsVisibleFile(path)
end

function META:Recompile(path, lol, diagnostics, on_save_path, target_state)
	if path then path = normalize_path(path) end

	on_save_path = on_save_path or path

	if on_save_path then on_save_path = normalize_path(on_save_path) end

	local cfg = self:GetCompilerConfig(path)
	diagnostics = diagnostics or {}
	local light_lsp_mode = cfg.lsp and (cfg.lsp.analyze == false or cfg.lsp.parse_only)
	local syntax_only = target_state == "syntax"

	if not path and light_lsp_mode then return true end

	if not path and not lol and not light_lsp_mode then
		if type(cfg.lsp.entry_point) == "table" then
			if self.debug then print("recompiling entry points from: " .. path) end

			local ok = true
			local reasons = {}

			for _, path in ipairs(cfg.lsp.entry_point) do
				local new_path = path_util.Resolve(path, cfg.parser.root_directory, cfg.parser.working_directory)

				if self.debug then
					print(path, "->", new_path)
					table.print(cfg)
				end

				local b, reason = self:Recompile(new_path, true, diagnostics, on_save_path)

				if not b then
					ok = false
					table.insert(reasons, reason)
				end
			end

			if path and not self:IsLoaded(path) then
				local b, reason = self:Recompile(path, true, diagnostics, on_save_path)

				if not b then
					ok = false
					table.insert(reasons, reason)
				end
			end

			return ok, table.concat(reasons, "\n")
		elseif type(cfg.lsp.entry_point) == "string" then
			local path = path_util.Resolve(cfg.lsp.entry_point, cfg.parser.root_directory, cfg.parser.working_directory)

			if self.debug then
				print("recompiling entry point: " .. path)
				print(cfg.lsp.entry_point, "->", path)
				table.print(cfg)
			end

			local ok, reason = self:Recompile(path, true, diagnostics, on_save_path)
			return ok, reason
		end
	end

	local entry_point = path or cfg.lsp.entry_point

	if not entry_point then return false, "no entry point" end

	if light_lsp_mode then
		entry_point = path or entry_point
		cfg.parser.import_validation_only = cfg.parser.import_validation_only ~= false
	end

	if syntax_only then
		entry_point = path or entry_point
		cfg.parser.skip_import = true
		cfg.parser.import_validation_only = false
	end

	cfg.parser.pre_read_file = function(parser, path)
		path = path_util.Normalize(path)

		if self.TempFiles[path] then return self:GetFileContent(path) end

		if path:sub(1, 1) ~= "/" then
			local workspace_path = path_util.Normalize(self.WorkingDirectory .. "/" .. path)

			if self.TempFiles[workspace_path] then
				return self:GetFileContent(workspace_path)
			end
		end

		return fs.read(path)
	end
	cfg.analyzer.pre_read_file = function(parser, path)
		if self.TempFiles[path] then return self:GetFileContent(path) end

		return fs.read(path)
	end
	cfg.analyzer.on_read_file = function(parser, path, content)
		if not self.TempFiles[path] then self:SetFileContent(path, content) end
	end
	cfg.parser.on_parsed_file = function(path, compiler)
		self:LoadFile(path, compiler.Code, compiler.Tokens, compiler)
	end
	cfg.parser.inline_require = not light_lsp_mode and not syntax_only
	self:DebugLog("[ " .. entry_point .. " ] compiling")
	local compiler

	if path == entry_point or entry_point == on_save_path then
		local content = self.TempFiles[entry_point] or fs.read(entry_point)
		compiler = Compiler(content or "", entry_point, cfg)
	else
		compiler = Compiler([[return import("]] .. entry_point .. [[")]], entry_point, cfg)
	end

	compiler:SetEnvironments(self:GetEnvironment())

	function compiler.OnDiagnostic(_, code, msg, severity, start, stop, node, level, ...)
		local name = code:GetName()
		local str_msg = formating.FormatMessage(msg, ...)

		if severity == "fatal" then
			self:DebugLog("[ " .. entry_point .. " ] " .. str_msg)
		end

		diagnostics[name] = diagnostics[name] or {}
		table.insert(
			diagnostics[name],
			{
				severity = severity,
				code = code,
				start = start,
				stop = stop,
				message = str_msg,
				trace = callstack.traceback(),
			}
		)
	end

	local ok, err = compiler:Parse()

	if ok then
		if compiler.SyntaxTree.imports then
			for _, root_node in ipairs(compiler.SyntaxTree.imports) do
				self:Yield()
				local root = root_node.RootStatement

				if root_node.RootStatement then
					if not root_node.RootStatement.parser then
						root = root_node.RootStatement.RootStatement
					end

					-- if root is false it failed to import and will be reported shortly after
					if root then
						self:SetFileContent(root.parser.config.file_path, root.code:GetString())
						self:LoadFile(root.parser.config.file_path, root.code, root.lexer_tokens, root.parser.compiler or compiler)
						diagnostics[root.parser.config.file_path] = diagnostics[root.parser.config.file_path] or {}
					end
				end
			end
		end

		-- Always load the main file itself
		if path then
			self:SetFileContent(path, compiler.Code:GetString())
			self:LoadFile(path, compiler.Code, compiler.Tokens, compiler)
			diagnostics[path] = diagnostics[path] or {}
		end

		local should_analyze = false
		local lsp_analyze = not (cfg.lsp and (cfg.lsp.analyze == false or cfg.lsp.parse_only))
		local force_analyze = false

		if path then
			local content = self.TempFiles[path] or fs.read(path)
			force_analyze = has_force_analyze_directive(content)

			if force_analyze then
				should_analyze = true
			elseif not lsp_analyze then
				should_analyze = false
			else
				local ignored = false
				local project_cfg = self:GetProjectConfig("get-compiler-config", path)

				if project_cfg and project_cfg.ignorefiles then
					for _, pattern in ipairs(project_cfg.ignorefiles) do
						if path:find(pattern) then
							if path:find("test_focus%.nlua") then
								self:DebugLog("not ignoring test_focus.nlua")
							else
								ignored = true

								break
							end
						end
					end
				end

				if not ignored then
					if path:find("%.nlua$") then should_analyze = true end
				end
			end
		elseif lsp_analyze and cfg.lsp.entry_point then
			should_analyze = true
		end

		if target_state == "parsed" or syntax_only then should_analyze = false end

		if should_analyze then
			local restore_import_roots
			local restore_should_crawl_untyped_functions

			if
				force_analyze and
				not lsp_analyze and
				compiler.SyntaxTree and
				compiler.SyntaxTree.imports
			then
				local imports = {}

				for _, import_node in ipairs(compiler.SyntaxTree.imports) do
					imports[#imports + 1] = {node = import_node, root = import_node.RootStatement}
					import_node.RootStatement = false
				end

				restore_import_roots = function()
					for _, data in ipairs(imports) do
						data.node.RootStatement = data.root
					end
				end
			end

			if force_analyze and not lsp_analyze and cfg.analyzer then
				local previous = cfg.analyzer.should_crawl_untyped_functions
				cfg.analyzer.should_crawl_untyped_functions = false
				restore_should_crawl_untyped_functions = function()
					cfg.analyzer.should_crawl_untyped_functions = previous
				end
			end

			local ok, err = compiler:Analyze(nil, cfg.analyzer)

			if restore_import_roots then restore_import_roots() end

			if restore_should_crawl_untyped_functions then
				restore_should_crawl_untyped_functions()
			end

			local name = compiler:GetCode():GetName()

			if not ok then
				diagnostics[name] = diagnostics[name] or {}
				local node = cfg.analyzer and
					cfg.analyzer.GetCurrentExpression and
					(
						cfg.analyzer:GetCurrentExpression() or
						cfg.analyzer:GetCurrentStatement()
					)
				local start, stop = 1, compiler:GetCode():GetByteSize()

				if node then start, stop = node:GetStartStop() end

				table.insert(
					diagnostics[name],
					{
						severity = "fatal",
						code = compiler:GetCode(),
						start = start,
						stop = stop,
						message = err,
					}
				)
			end

			for typ, node in pairs(compiler.analyzer:GetTypeToNodeMap()) do
				self.node_to_type[typ] = node
			end

			self:MarkAnalyzed(name)

			if compiler.SyntaxTree.imports then
				for _, root_node in ipairs(compiler.SyntaxTree.imports) do
					self:Yield()
					local root = root_node.RootStatement

					if root then
						if not root.parser and root.RootStatement then
							root = root.RootStatement
						end

						if root and root.parser and root.parser.config and root.parser.config.file_path then
							self:MarkAnalyzed(root.parser.config.file_path)
						end
					end
				end
			end

			if
				on_save_path and
				cfg and
				(
					cfg.analyzer.remove_unused or
					(
						cfg.emitter and
						cfg.emitter.remove_unused
					)
				)
			then
				self:DebugLog("checking roots for on_save_path: " .. tostring(on_save_path))
				local roots = {}

				if compiler.SyntaxTree.code then
					self:DebugLog(
						"compiler.SyntaxTree.code:GetName() = " .. tostring(compiler.SyntaxTree.code:GetName())
					)

					if
						normalize_path(compiler.SyntaxTree.code:GetName()) == normalize_path(on_save_path)
					then
						self:DebugLog("found compiler.SyntaxTree match")
						table.insert(roots, compiler.SyntaxTree)
					end
				end

				if compiler.SyntaxTree.imports then
					for _, root_node in ipairs(compiler.SyntaxTree.imports) do
						local root = root_node.RootStatement

						if root_node.RootStatement then
							if not root_node.RootStatement.parser then
								root = root_node.RootStatement.RootStatement
							end

							if root and root.code then
								if normalize_path(root.code:GetName()) == normalize_path(on_save_path) then
									self:DebugLog("found import match: " .. tostring(root.code:GetName()))
									table.insert(roots, root)
								-- else
								-- 	self:DebugLog("not matching import: " .. tostring(root.code:GetName()))
								end
							end
						end
					end
				end

				for _, root in ipairs(roots) do
					self:DebugLog("REMOVING UNUSED IN: " .. tostring(root.code:GetName()))
					local ok, new_code = pcall(function()
						local cfg_copy = {}

						for k, v in pairs(cfg.emitter) do
							cfg_copy[k] = v
						end

						if cfg_copy.trailing_newline == nil then cfg_copy.trailing_newline = true end

						cfg_copy.remove_unused = true
						cfg_copy.skip_import = true
						cfg_copy.no_pipeline = true
						cfg_copy.comment_type_annotations = on_save_path:sub(-#".lua") == ".lua"
						local emitter = Emitter(cfg_copy)
						return emitter:BuildCode(root)
					end)

					if not ok then
						self:DebugLog("[ " .. entry_point .. " ] emitter failed: " .. new_code)
					elseif new_code ~= root.code:GetString() then
						local path = on_save_path

						if root.parser and root.parser.config then
							path = root.parser.config.file_path
						end

						fs.write(path, new_code)
						self:SetFileContent(path, new_code)
					end
				end
			end
		else
			self:DebugLog("[ " .. entry_point .. " ] skipped analysis")
		end
	end

	-- Add diagnostics for unreachable code (dimmed with Unnecessary tag)
	local file_data = self:IsLoaded(path) and self:GetFile(path)

	if file_data and file_data.compiler and file_data.compiler.SyntaxTree then
		local code_obj = file_data.code
		local diag_name = code_obj and code_obj:GetName()

		if diag_name then
			local function collect_unreachable(node)
				if not node then return end

				if node.statements then
					for _, child in ipairs(node.statements) do
						if type(child) == "table" then
							if child.Type then
								-- It's a statement node
								if child.Unreachable == true then
									local start, stop = child:GetStartStop()

									if start and stop then
										diagnostics[diag_name] = diagnostics[diag_name] or {}
										table.insert(
											diagnostics[diag_name],
											{
												severity = "hint",
												code = code_obj,
												start = start,
												stop = stop,
												message = "unreachable code",
												unreachable = true,
											}
										)
									end
								end

								collect_unreachable(child)
							else
								-- It's an array of statements (e.g., if branch)
								for _, stmt in ipairs(child) do
									if type(stmt) == "table" and stmt.Type then
										if stmt.Unreachable == true then
											local start, stop = stmt:GetStartStop()

											if start and stop then
												diagnostics[diag_name] = diagnostics[diag_name] or {}
												table.insert(
													diagnostics[diag_name],
													{
														severity = "hint",
														code = code_obj,
														start = start,
														stop = stop,
														message = "unreachable code",
														unreachable = true,
													}
												)
											end
										end

										collect_unreachable(stmt)
									end
								end
							end
						end
					end
				end
			end

			collect_unreachable(file_data.compiler.SyntaxTree)
		end
	end

	local next_published = {}
	local normalized_path = normalize_path(path)
	local normalized_entry_point = normalize_path(entry_point)
	local normalized_on_save_path = normalize_path(on_save_path)

	local function should_publish(name)
		name = normalize_path(name)
		return self.OpenFiles[name] or
			(
				normalized_path and
				name == normalized_path
			)
			or
			(
				normalized_entry_point and
				name == normalized_entry_point
			)
			or
			(
				normalized_on_save_path and
				name == normalized_on_save_path
			)
	end

	for name, data in pairs(diagnostics) do
		self:Yield()

		if should_publish(name) then
			next_published[normalize_path(name)] = true

			if #data > 0 then
				self:OnDiagnostics(name, data)
			else
				self:OnClearDiagnostics(name)
			end
		end
	end

	for name in pairs(self.PublishedDiagnostics) do
		self:Yield()

		if not next_published[name] and not self.OpenFiles[name] then
			self:OnClearDiagnostics(name)
		end
	end

	for name in pairs(self.OpenFiles) do
		if not next_published[name] and diagnostics[name] and #diagnostics[name] == 0 then
			next_published[name] = true
		end
	end

	self.PublishedDiagnostics = next_published

	if path then self:ClearDirty(path) end

	return true
end

function META:GetEnvironment()
	return runtime_env, typesystem_env
end

function META:OnDiagnostics(name, data) end

function META:OnClearDiagnostics(name) end

function META:OnResponse(response) end

function META:Initialize()
	local ok, reason = self:Recompile()

	if not ok and reason ~= "no entry point" then
		if not _G.TEST then print(":Recompile() failed: " .. reason) end
	end
end

function META:Format(code, path, extra_emitter_config)
	path = path_util.Normalize(path)
	local config = self:GetCompilerConfig(path)
	apply_format_emitter_defaults(config.emitter)
	config.parser = {skip_import = true}
	config.emitter.comment_type_annotations = path:sub(-#".lua") == ".lua"
	config.emitter.transpile_extensions = path:sub(-#".lua") == ".lua"
	config.emitter.skip_import = true
	config.emitter.no_pipeline = true
	-- remove_unused should only be triggered by explicit request (e.g. code action),
	-- not by the workspace-level removeUnusedOnSave setting (that goes through the Recompile save path)
	local do_remove_unused = extra_emitter_config and extra_emitter_config.remove_unused
	config.emitter.remove_unused = do_remove_unused or false
	config.analyzer.remove_unused = do_remove_unused or false
	local file_data = self:IsLoaded(path) and self:GetFile(path)
	local loaded_parser_cfg = file_data and
		file_data.compiler and
		file_data.compiler.Config and
		file_data.compiler.Config.parser or
		{}
	local can_reuse_loaded_tree = file_data and
		file_data.compiler and
		file_data.code:GetString() == code and
		loaded_parser_cfg.skip_import == config.parser.skip_import

	if can_reuse_loaded_tree then
		local cfg_copy = {}

		for k, v in pairs(config.emitter) do
			cfg_copy[k] = v
		end

		if do_remove_unused then file_data.compiler:Analyze() end

		local emitter = Emitter(cfg_copy)

		if extra_emitter_config then
			for k, v in pairs(extra_emitter_config) do
				emitter.config[k] = v
			end
		end

		return emitter:BuildCode(file_data.compiler.SyntaxTree)
	end

	local compiler = Compiler(code, "@" .. path, config)

	if do_remove_unused then compiler:Analyze() end

	local code, err = compiler:Emit()
	return code
end

function META:OpenFile(path, code, should_recompile)
	path = path_util.Normalize(path)
	self.OpenFiles[path] = true
	self:SetFileContent(path, code)

	if should_recompile == nil then
		should_recompile = self:ShouldRecompileOnOpen(path)
	end

	if should_recompile and not self:IsLightLspMode(path) then
		assert(self:Recompile(path))
	end
end

function META:CloseFile(path)
	path = path_util.Normalize(path)
	self.OpenFiles[path] = nil
	self.PublishedDiagnostics[path] = nil
	self:ClearDirty(path)
	self:SetFileContent(path, nil)
	self:UnloadFile(path)
	self:OnClearDiagnostics(path)
end

function META:UpdateFile(path, code)
	path = path_util.Normalize(path)
	self:SetFileContent(path, code)
	self:MarkDirty(path)
end

function META:SaveFile(path)
	path = path_util.Normalize(path)
	local cfg = self:GetCompilerConfig(path)
	local content = self.TempFiles[path] or fs.read(path) or ""
	local needs_full_save_refresh = has_force_analyze_directive(content) or
		(
			cfg.analyzer and
			cfg.analyzer.remove_unused
		)
		or
		(
			cfg.emitter and
			cfg.emitter.remove_unused
		)
	local target_state = "syntax"

	if needs_full_save_refresh then target_state = nil end

	self:SetFileContent(path, nil)
	assert(self:Recompile(path, nil, nil, path, target_state))
end

function META:EnsureLoaded(path)
	return self:EnsureAnalyzed(path)
end

function META:EnsureParsed(path)
	path = path_util.Normalize(path)

	if self:IsParsed(path) and not self:IsDirty(path) then return true end

	local ok, err = self:Recompile(path, nil, nil, nil, "parsed")

	if not ok then
		self:DebugLog("[ " .. path .. " ] EnsureParsed failed: " .. tostring(err))
		return false
	end

	return self:IsParsed(path)
end

function META:EnsureAnalyzed(path)
	path = path_util.Normalize(path)

	if self:IsDirty(path) then
		local ok, err = self:Recompile(path)

		if not ok then
			self:DebugLog("[ " .. path .. " ] EnsureAnalyzed failed: " .. tostring(err))
			return false
		end
	end

	if self:IsAnalyzed(path) then return true end

	if self:IsLightLspMode(path) then
		return self:IsParsed(path) or self:EnsureParsed(path)
	end

	local ok, err = self:Recompile(path)

	if not ok then
		self:DebugLog("[ " .. path .. " ] EnsureAnalyzed failed: " .. tostring(err))
		return false
	end

	return self:IsAnalyzed(path)
end

function META:GetAllTokens(path)
	local data = self:GetFile(path)
	return data.tokens
end

function META:FindToken(path, line, char)
	line = line + 1
	char = char + 1
	local data = self:GetFile(path)

	if not data then return end

	local sub_pos = data.code:LineCharToSubPos(line, char)

	for i, token in ipairs(data.tokens) do
		if sub_pos >= token.start and sub_pos <= token.stop then
			return token, data
		end
	end

	-- Fallback to nearby token (e.g. if cursor is on whitespace)
	for i = 1, 5 do
		local offset = i

		for _, dir in ipairs({-1, 1}) do
			local p = sub_pos + offset * dir

			for _, token in ipairs(data.tokens) do
				if p >= token.start and p <= token.stop then return token, data end
			end
		end
	end
end

function META:FindTokensFromRange(
	path--[[#: string]],
	line_start--[[#: number]],
	char_start--[[#: number]],
	line_stop--[[#: number]],
	char_stop--[[#: number]]
)
	line_start = line_start + 1
	char_start = char_start + 1
	line_stop = line_stop + 1
	char_stop = char_stop + 1
	local data = self:GetFile(path)

	if not data then return {} end

	local sub_pos_start = data.code:LineCharToSubPos(line_start, char_start)
	local sub_pos_stop = data.code:LineCharToSubPos(line_stop, char_stop)
	local found = {}

	for _, token in ipairs(data.tokens) do
		if token.start >= sub_pos_start and token.stop <= sub_pos_stop then
			table.insert(found, token)
		end
	end

	return found
end

do
	local function find_parent(token, typ)
		local node = token.parent

		if not node then return nil end

		while node.parent do
			if type(typ) == "function" then
				if typ(node) then return node end
			else
				if node.Type == typ then return node end
			end

			node = node.parent
		end

		return nil
	end

	local function find_nodes(tokens, type)
		local nodes = {}
		local done = {}

		for _, token in ipairs(tokens) do
			local node = find_parent(token, type)

			if node and not done[node] then
				table.insert(nodes, node)
				done[node] = true
			end
		end

		return nodes
	end

	function META:GetInlayHints(path, start_line, start_character, stop_line, stop_character)
		local tokens = self:FindTokensFromRange(path, start_line, start_character, stop_line, stop_character)
		local hints = {}
		local assignments = find_nodes(tokens, "statement_local_assignment")

		for _, assingment in ipairs(find_nodes(tokens, "statement_assignment")) do
			table.insert(assignments, assingment)
		end

		for _, assignment in ipairs(assignments) do
			if assignment.environment == "runtime" then
				for i, left in ipairs(assignment.left) do
					if not left.tokens[":"] and assignment.right and assignment.right[i] then
						local types = left:GetAssociatedTypes()

						if
							types and
							#types > 0 and
							(
								assignment.right[i].Type ~= "expression_value" or
								assignment.right[i].value.value.type == "letter"
							)
						then
							local label

							do
								local filtered = {}

								for _, t in ipairs(types) do
									if t.Type == "upvalue" then
										table.insert(filtered, t:GetValue())
									else
										table.insert(filtered, t)
									end
								end

								if #filtered == 1 then
									label = tostring(filtered[1])
								elseif #filtered > 1 then
									label = tostring(Union(filtered))
								else
									label = ""
								end

								if label:sub(-1) == "|" then label = label:sub(1, -2) end
							end

							if #label > 0 then
								if #label > 20 then label = label:sub(1, 20) .. "..." end

								local start, stop = left:GetStartStop()
								table.insert(
									hints,
									{
										label = label,
										start = start,
										stop = stop,
									}
								)
							end
						end
					end
				end
			end
		end

		return hints
	end

	function META:GetScopes(path)
		local tokens = self:GetAllTokens(path)
		local statements = find_nodes(tokens, function(node)
			if node.scopes then return node end
		end)
		local scopes = {}

		for _, statement in ipairs(statements) do
			for i, scope in ipairs(statement.scopes) do
				table.insert(scopes, {scope = scope, statement = statement})
			end
		end

		return scopes
	end
end

function META:GetCode(path)
	local data = self:GetFile(path)
	return data.code
end

local function create_source_node(path, content, start, stop)
	return {
		Code = Code(content, path),
		GetStartStop = function()
			return start, stop
		end,
		GetSourcePath = function()
			return path
		end,
	}
end

local function find_builtin_definition_node(token)
	if not token then return nil end

	local root_name = token:GetValueString()
	local member_name
	local value_node = token.parent
	local expression = value_node and value_node.parent

	if
		expression and
		expression.Type == "expression_binary_operator" and
		expression.value and
		expression.value.value == "." and
		expression.right == value_node and
		expression.left and
		expression.left.value
	then
		root_name = expression.left.value:GetValueString()
		member_name = token:GetValueString()
	end

	if not root_name or not root_name:match("^[%a_][%w_]*$") then return nil end

	local file_candidates = {
		"nattlua/definitions/lua/" .. root_name .. ".nlua",
		"nattlua/definitions/lua/globals.nlua",
		"nattlua/definitions/utility.nlua",
	}
	local search_specs = {}

	if member_name then
		local qualified_name = root_name .. "." .. member_name
		table.insert(
			search_specs,
			{anchor = "analyzer function " .. qualified_name, symbol = member_name}
		)
		table.insert(search_specs, {anchor = "function " .. qualified_name, symbol = member_name})
		table.insert(
			search_specs,
			{anchor = "type Modules[\"" .. qualified_name .. "\"]", symbol = qualified_name}
		)
		table.insert(
			search_specs,
			{anchor = "type Modules['" .. qualified_name .. "']", symbol = qualified_name}
		)
	else
		table.insert(search_specs, {anchor = "type " .. root_name .. " =", symbol = root_name})
		table.insert(search_specs, {anchor = "local " .. root_name .. " =", symbol = root_name})
		table.insert(search_specs, {anchor = "function " .. root_name, symbol = root_name})
	end

	for _, builtin_path in ipairs(file_candidates) do
		local content = fs.read(builtin_path)

		if content then
			for _, spec in ipairs(search_specs) do
				local anchor_start = content:find(spec.anchor, 1, true)

				if anchor_start then
					local symbol_start = content:find(spec.symbol, anchor_start, true)

					if symbol_start then
						return create_source_node(
							builtin_path,
							content,
							symbol_start,
							symbol_start + #spec.symbol - 1
						)
					end
				end
			end
		end
	end

	return nil
end

local function maybe_remap_bundled_builtin_definition(node, token)
	local source_path = node and node.GetSourcePath and node:GetSourcePath()

	if source_path and source_path:find("nattlua/definitions/index.nlua", 1, true) then
		return find_builtin_definition_node(token) or node
	end

	if not node then return find_builtin_definition_node(token) end

	return node
end

function META:GetRenameInstructions(path, line, character, newName)
	local token, data = self:FindToken(path, line, character)

	if not token then return end

	local str = token:GetValueString()
	local upvalue = token:FindUpvalue()
	local edits = {}

	if not upvalue then
		table.insert(
			edits,
			{
				start = token.start,
				stop = token.stop,
				from = token.value,
				to = newName,
			}
		)
		return edits
	end

	for i, v in ipairs(data.tokens) do
		local u = v:FindUpvalue()

		if u and v.type == "letter" and v:GetValueString() == str then
			if u == upvalue or u:GetKey() == upvalue:GetKey() then
				table.insert(
					edits,
					{
						start = v.start,
						stop = v.stop,
						from = str,
						to = newName,
					}
				)
			end
		end
	end

	return edits
end

function META:GetDefinition(path, line, character)
	local token = self:FindToken(path, line, character)

	if not token then return end

	local function return_definition(node)
		local mapped = maybe_remap_bundled_builtin_definition(node, token)

		if mapped and mapped.GetStartStop then return mapped end

		return nil
	end

	local all_types = token:FindType()

	if not all_types[1] then return return_definition(nil) end

	local types = {}

	for _, t in ipairs(all_types) do
		if t.Type == "union" then
			for _, v in ipairs(t:GetData()) do
				table.insert(types, v)
			end
		else
			table.insert(types, t)
		end
	end

	for _, typ in ipairs(types) do
		do
			local node = self:NodeToType(typ)

			if node and node.GetStartStop then return return_definition(node) end
		end

		if typ.Type == "upvalue" then
			local node = self:NodeToType(typ)

			if node and node.GetStartStop then return return_definition(node) end

			typ = typ:GetValue()

			if typ.Type == "union" then
				for _, v in ipairs(typ:GetData()) do
					table.insert(types, v)
				end

				goto next_type
			end

			local node = self:NodeToType(typ)

			if node and node.GetStartStop then return return_definition(node) end
		end

		if typ.Type == "function" then
			local node = typ:GetFunctionBodyNode()

			if node and node.GetStartStop then return return_definition(node) end
		end

		do
			local upvalue = typ.GetUpvalue and typ:GetUpvalue()

			if upvalue then
				local node = self:NodeToType(upvalue)

				if node and node.GetStartStop then return return_definition(node) end
			end
		end

		::next_type::
	end

	return return_definition(nil)
end

function META:GetSignatureHelp(path, line, character)
	local token, data = self:FindToken(path, line, character)

	if not token then return end

	local function find_call_node(node)
		while node do
			if
				node.Type == "expression_call" or
				node.Type == "statement_call" or
				node.Type == "expression_postfix_call"
			then
				return node
			end

			node = node.parent
		end
	end

	local call_node = find_call_node(token.parent)

	if not call_node then return end

	local types = call_node.left:GetAssociatedTypes()
	local func_type

	for _, t in ipairs(types) do
		if t.Type == "function" then
			func_type = t

			break
		end
	end

	if not func_type then return end

	local signatures = {}
	local active_param = 0

	-- Calculate active parameter based on comma counts or current args
	if call_node.expressions then
		for i, arg in ipairs(call_node.expressions) do
			local start, stop = arg:GetStartStop()

			if start and stop then
				local sub_pos = data.code:LineCharToSubPos(line + 1, character + 1)

				if sub_pos > stop then active_param = i end
			end
		end
	end

	local func_name = call_node.left:Render()
	local input = func_type:GetInputSignature()
	local params = {}
	local label = func_name .. "("

	if input then
		for i, param in ipairs(input:GetData()) do
			local param_name = "arg" .. i
			local param_label = tostring(param)
			table.insert(params, {label = param_label})
			label = label .. param_label .. (i < #input:GetData() and ", " or "")
		end
	end

	label = label .. ")"
	table.insert(signatures, {
		label = label,
		parameters = params,
	})
	return {
		signatures = signatures,
		activeSignature = 0,
		activeParameter = active_param,
	}
end

function META:GetHighlightRanges(path)
	-- find the .coverage file
	local directory = path_util.GetDirectory(path)
	local name = path_util.GetFileName(path)
	local coverage_file = path_util.Join(directory, name .. ".coverage")

	if not fs.is_file(coverage_file) then return end

	local data = loadstring(fs.read(coverage_file))()
	local max_count = 1

	for _, item in ipairs(data) do
		max_count = math.max(max_count, item[3] or 1)
	end

	local ranges = {}

	for _, item in ipairs(data) do
		local count = item[3]

		if count > 0 then
			local normalized = count / max_count
			local r = math.floor(normalized * 255)
			local g = math.floor((1 - normalized) * 255)
			local b = 0
			local color = string.format("#%02x%02x%02x1A", r, g, b)
			table.insert(
				ranges,
				{
					start = item[1],
					stop = item[2],
					backgroundColor = color,
				}
			)
		end
	end

	return ranges
end

function META:GetUpvalueHighlightRanges(path, line, character)
	local token, data = self:FindToken(path, line, character)

	if not token then return end

	local upvalue = token:FindUpvalue()

	if not upvalue then return end

	local ranges = {}

	for i, v in ipairs(data.tokens) do
		local u = v:FindUpvalue()

		if u == upvalue and v.type == "letter" then
			local start, stop = v.start, v.stop
			table.insert(ranges, self:GetCode(path):SubPosToLineChar(start, stop))
		end
	end

	return ranges
end

function META:GetHover(path, line, character)
	local token = self:FindToken(path, line, character)

	if not token then return end

	local types, found_parents, scope = token:FindType()
	local obj

	if #types > 0 then
		local filtered = {}

		for _, t in ipairs(types) do
			if t.Type ~= "upvalue" then table.insert(filtered, t) end
		end

		if #filtered == 1 then
			obj = filtered[1]
		elseif #filtered > 1 then
			obj = Union(filtered)
		else
			obj = types[1]
		end
	end

	return {
		obj = obj,
		scope = scope,
		found_parents = found_parents,
		token = token,
	}
end

do
	local runtime_syntax = require("nattlua.syntax.runtime")
	local LString = require("nattlua.types.string").LString

	local function tostring_key(obj)
		if obj.Type == "string" and obj:IsLiteral() then return obj:GetData() end

		return tostring(obj)
	end

	local function tostring_val(obj)
		local str = tostring(obj)

		if #str > 100 then return str:sub(1, 100) .. "..." end

		return str
	end

	local function get_key_values(self, obj, scope, runtime)
		if not obj or obj.Type == "any" then
			local r, t = self:GetEnvironment()
			local tbl = {}

			for key, data in pairs(get_key_values(self, r, scope, runtime)) do
				tbl[key] = data
			end

			for key, data in pairs(get_key_values(self, t, scope, runtime)) do
				tbl[key] = data
			end

			for keyword in pairs(runtime_syntax.Keywords) do
				tbl[keyword] = {val = keyword, obj = "keyword"}
			end

			for keyword in pairs(runtime_syntax.NonStandardKeywords) do
				tbl[keyword] = {val = keyword, obj = "keyword"}
			end

			if scope then
				for key, upvalue in pairs(scope:GetAllVisibleUpvalues()) do
					tbl[key] = {val = key, obj = upvalue}
				end
			end

			for _, typ in ipairs{"string", "number", "any", "true", "nil", "false"} do
				tbl[typ] = {val = typ, obj = "keyword"}
			end

			return tbl
		end

		if obj.Type == "table" then
			local out = {}

			for _, kv in ipairs(obj:GetData()) do
				local key = tostring_key(kv.key)
				local val = tostring_val(kv.val)
				out[key] = {val = val, obj = kv.val}
			end

			if obj:GetContract() and obj:GetContract() ~= obj then
				for _, kv in ipairs(obj:GetContract():GetData()) do
					local key = tostring_key(kv.key)
					local val = tostring_val(kv.val)
					out[key] = {val = val, obj = kv.val}
				end
			end

			if obj:GetMetaTable() then
				local t = obj:GetMetaTable():Get(LString("__index"))

				if t and t.Type == "table" and t ~= obj then
					for _, kv in ipairs(t:GetData()) do
						local key = tostring_key(kv.key)
						local val = tostring_val(kv.val)
						out[key] = {val = val, obj = kv.val}
					end
				end
			end

			return out
		elseif obj.Type == "string" then
			return get_key_values(self, obj:GetMetaTable():Get(LString("__index")), scope, runtime)
		elseif obj.Type == "tuple" then
			if runtime then
				return get_key_values(self, obj:GetFirstValue(), scope, runtime)
			end
		elseif obj.Type == "union" then
			local out = {}

			for _, obj in ipairs(obj:GetData()) do
				for key, data in pairs(get_key_values(self, obj, scope, runtime)) do
					out[key] = data
				end
			end

			return out
		end

		return nil
	end

	function META:GetKeyValuesForCompletion(path, line, character)
		local data = self:GetHover(path, line, character)
		print(data.token, data.obj, "<< autocompleting")
		return get_key_values(self, data and data.obj, data.scope, false), data
	end
end

function META:GetReferences(path, line, character)
	local token, data = self:FindToken(path, line, character)

	if not token then return {} end

	local upvalue = token:FindUpvalue()

	if upvalue then
		local references = {}

		for path, data in pairs(self.LoadedFiles) do
			for _, v in ipairs(data.tokens) do
				if
					v:FindUpvalue() == upvalue and
					v.type == "letter" and
					v:GetValueString() == token:GetValueString()
				then
					table.insert(references, v)
				end
			end
		end

		if #references > 0 then return references end
	end

	-- Fallback for non-upvalue (could be a type from node_to_type mapping)
	local types = token:FindType()
	local nodes = {}

	for _, obj in ipairs(types) do
		local node
		local upvalue = (obj.Type == "upvalue" and obj) or (obj.GetUpvalue and obj:GetUpvalue())

		if upvalue then
			node = self:NodeToType(upvalue)
		elseif obj.GetFunctionBodyNode and obj:GetFunctionBodyNode() then
			node = obj:GetFunctionBodyNode()
		elseif self:NodeToType(obj) then
			node = self:NodeToType(obj)
		end

		if node then table.insert(nodes, node) end
	end

	return nodes
end

do
	local function build_ast(self, path, node, done)
		done = done or {}

		if done[node] then return end

		done[node] = true

		if type(node) ~= "table" or not node.Type then return end

		local root = {
			name = node.Type,
			detail = tostring(node),
			kind = "Variable",
			children = {},
		}

		for k, v in pairs(node) do
			if type(v) == "table" then
				local child = build(self, path, v, done)

				if child then
					table.insert(root.children, child)
				else
					for i, v in ipairs(v) do
						if type(node) ~= "table" or not node.Type then

						else
							local child = build(self, path, v, done)

							if child then table.insert(root.children, child) end
						end
					end
				end
			end
		end

		return root
	end

	local function build_types(self, path, node, obj, env, done)
		done = done or {}

		if done[obj] then
			return {
				name = "*recursive*",
				kind = "Variable",
				children = {},
			}
		end

		done[obj] = true

		if obj.Type == "lexical_scope" then
			local root = {
				name = tostring(obj),
				kind = "Module",
				children = {},
			}

			for _, upvalue in ipairs(obj.upvalues.runtime.list) do
				table.insert(root.children, build(self, path, node, upvalue, "runtime", done))
			end

			for _, upvalue in ipairs(obj.upvalues.typesystem.list) do
				table.insert(root.children, build(self, path, node, upvalue, "typesystem", done))
			end

			for _, obj in ipairs(obj:GetChildren()) do
				table.insert(root.children, build(self, path, node, obj, env, done))
			end

			return root
		elseif obj.Type == "upvalue" then
			local val = obj:GetMutatedValue(obj:GetScope())
			local node2 = obj.statement or node
			local root = {
				name = obj.Key,
				kind = env == "runtime" and "Variable" or "TypeParameter",
				children = {build(self, path, node, val, env, done)},
			}
			return root
		elseif obj.Type == "symbol" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Enum",
				children = {},
			}
			return root
		elseif obj.Type == "union" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Namespace",
				children = {},
			}

			for _, obj in ipairs(obj:GetData()) do
				table.insert(root.children, build(self, path, node, obj, env, done))
			end

			return root
		elseif obj.Type == "tuple" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Array",
				children = {},
			}

			for _, obj in ipairs(obj:GetData()) do
				table.insert(root.children, build(self, path, node, obj, env, done))
			end

			return root
		elseif obj.Type == "table" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Array",
				children = {},
			}

			for _, kv in ipairs(obj:GetData()) do
				local field = {
					name = tostring(kv.key) .. " = " .. tostring(kv.val),
					detail = tostring(node),
					kind = "Variable",
					children = {
						build(self, path, node, kv.key, env, done),
						build(self, path, node, kv.val, env, done),
					},
				}
				table.insert(root.children, field)
			end

			return root
		elseif obj.Type == "number" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Number",
				children = {},
			}
			return root
		elseif obj.Type == "range" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Number",
				children = {},
			}
			return root
		elseif obj.Type == "string" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "String",
				children = {},
			}
			return root
		elseif obj.Type == "any" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "TypeParameter",
				children = {},
			}
			return root
		elseif obj.Type == "function" then
			local node2 = obj.statement or node
			local root = {
				name = tostring(obj),
				kind = "Function",
				children = {},
			}
			table.insert(root.children, build(self, path, node, obj:GetInputSignature(), env, done))
			table.insert(root.children, build(self, path, node, obj:GetOutputSignature(), env, done))
			return root
		else
			error("nyi type: " .. obj.Type)
		end
	end

	local function build_scopes(self, path, node, obj, env)
		if obj.Type == "lexical_scope" then
			local root = {
				name = tostring(obj),
				kind = "Module",
				range = get_range(self:GetCode(path), (obj.node or node):GetStartStop()),
				selectionRange = get_range(self:GetCode(path), (obj.node or node):GetStartStop()),
				children = {},
			}

			for _, upvalue in ipairs(obj.upvalues.runtime.list) do
				table.insert(root.children, build_scopes(self, path, node, upvalue, "runtime", done))
			end

			for _, upvalue in ipairs(obj.upvalues.typesystem.list) do
				table.insert(root.children, build_scopes(self, path, node, upvalue, "typesystem", done))
			end

			for _, obj in ipairs(obj:GetChildren()) do
				table.insert(root.children, build_scopes(self, path, node, obj, env, done))
			end

			return root
		elseif obj.Type == "upvalue" then
			local val = obj:GetMutatedValue(obj:GetScope())
			local node2 = obj.statement or node
			local root = {
				name = obj.Key .. " = " .. tostring(val),
				kind = env == "runtime" and "Variable" or "TypeParameter",
				children = {},
			}
			return root
		else
			error("nyi type: " .. obj.Type)
		end
	end

	local function build_nodes(self, path, node)
		if not node then return {} end

		local symbols = {}

		local function normalize_symbol_name(name)
			if name == nil then return nil end

			name = tostring(name)
			name = name:match("^%s*(.-)%s*$") or name

			if name == "" then return nil end

			return name
		end

		local function add_symbol(name, kind, n, parent_symbols)
			name = normalize_symbol_name(name)

			if not name then return end

			local start, stop = n:GetStartStop()

			if not start then return end

			local code = self:GetCode(path)

			if not code then return end

			local range_data = code:SubPosToLineChar(start, stop)
			local symbol = {
				name = name,
				kind = kind,
				range = {
					start = {line = range_data.line_start - 1, character = range_data.character_start - 1},
					["end"] = {line = range_data.line_stop - 1, character = range_data.character_stop},
				},
				selectionRange = {
					start = {line = range_data.line_start - 1, character = range_data.character_start - 1},
					["end"] = {line = range_data.line_stop - 1, character = range_data.character_stop},
				},
				children = {},
			}
			table.insert(parent_symbols or symbols, symbol)
			return symbol
		end

		local function walk(n, current_symbols)
			if not n.GetNodes then return end

			for _, child in ipairs(n:GetNodes()) do
				local symbol

				if
					child.Type == "statement_local_function" or
					child.Type == "statement_global_function"
				then
					symbol = add_symbol(
						child.tokens["identifier"] and
							child.tokens["identifier"]:GetValueString() or
							"function",
						"Function",
						child,
						current_symbols
					)
				elseif
					child.Type == "statement_local_assignment" or
					child.Type == "statement_assignment"
				then
					if child.left then
						for _, expr in ipairs(child.left) do
							symbol = add_symbol(expr:Render(), "Variable", child, current_symbols)
						end
					end
				elseif child.Type == "statement_local_type_function" then
					symbol = add_symbol(
						child.tokens["identifier"] and
							child.tokens["identifier"]:GetValueString() or
							"type function",
						"Function",
						child,
						current_symbols
					)
				end

				if child.statements and type(child.statements) == "table" then
					walk(child, symbol and symbol.children or current_symbols)
				end
			end
		end

		walk(node, symbols)
		return symbols
	end

	function META:GetSymbolTree(path)
		local data = self:GetFile(path)

		if not data or not data.tokens or #data.tokens == 0 then return {} end

		local root_node = data.tokens[1].parent

		while root_node and root_node.parent do
			root_node = root_node.parent
		end

		if not root_node then return {} end

		return build_nodes(self, path, root_node)
	end
end

do
	local tokenTypeMap = {}
	local tokenModifiersMap = {}
	local SemanticTokenTypes = {
		-- identifiers or reference
		"class", -- a class type. maybe META or Meta?
		"typeParameter", -- local type >foo< = true
		"parameter", -- function argument: function foo(>a<)
		"variable", -- a local or global variable.
		"property", -- a member property, member field, or member variable.
		"enumMember", -- an enumeration property, constant, or member. uppercase variables and global non tables? local FOO = true ?
		"event", --  an event property.
		"function", -- local or global function: local function >foo<
		"method", --  a member function or method: string.>bar<()
		"type", -- misc type
		-- tokens
		"comment", -- 
		"string", -- 
		"keyword", -- 
		"number", -- 
		"regexp", -- regular expression literal.
		"operator", --
		"decorator", -- decorator syntax, maybe for @Foo in tables, $ and §
		-- other identifiers or references
		"namespace", -- namespace, module, or package.
		"enum", -- 
		"interface", --
		"struct", -- 
		"decorator", -- decorators and annotations.
		"macro", --  a macro.
		"label", --  a label. ??
	}
	local SemanticTokenModifiers = {
		"declaration", -- For declarations of symbols.
		"definition", -- For definitions of symbols, for example, in header files.
		"readonly", -- For readonly variables and member fields (constants).
		"static", -- For class members (static members).
		"private", -- For class members (static members).
		"deprecated", -- For symbols that should no longer be used.
		"abstract", -- For types and member functions that are abstract.
		"async", -- For functions that are marked async.
		"modification", -- For variable references where the variable is assigned to.
		"documentation", -- For occurrences of symbols in documentation.
		"defaultLibrary", -- For symbols that are part of the standard library.
	}

	for i, v in ipairs(SemanticTokenTypes) do
		tokenTypeMap[v] = i - 1
	end

	for i, v in ipairs(SemanticTokenModifiers) do
		tokenModifiersMap[v] = i - 1
	end

	local types = {
		keyword = "keyword",
		comment = "comment",
		operator = "keyword",
		type = "type",
		symbol = "enumMember",
		number = "number",
		string = "string",
		variable = "variable",
		any = "regexp",
		table = "class",
		func = "function",
	}

	local function get_semantic_type(token)
		if token.type == "fake" then return "string" end

		if token.type == "end_of_file" or token.type == "space" then return end

		if token.type == "multiline_comment" or token.type == "line_comment" then
			return types.comment
		end

		if token:IsKeywordValue() then return types.symbol end

		if token:IsKeyword() then return types.keyword end

		if token:IsUnreachable() then return types.any, {"deprecated"} end

		if token:IsOperator() then return types.operator end

		if token:IsString() then
			return types.string
		elseif token:IsNumber() then
			return types.number
		elseif token:IsTable() then
			return types.table
		elseif token:IsFunction() then
			return types.func
		elseif token:IsSymbol() then
			return types.symbol
		elseif token:IsOtherType() then
			return types.type
		end

		if token:IsAny() then return types.any end

		return types.variable
	end

	local function is_probably_lua(str)
		local possible_statements = {
			"^%s*local%s",
			"^%s*return%s",
			"^%s*while%s",
			"^%s*repeat%s",
			"^%s*function%s",
			"^%s*do%s",
			"^%s*if%s",
			"^%s*for%s",
			"^%s*[a-zA-Z_][a-zA-Z0-9_]*%s*=%s*",
		}

		for _, word in ipairs(possible_statements) do
			if str:find(word) then return true end
		end

		return false
	end

	function META:GetSemanticTokens(path)
		if not self:IsLoaded(path) then return {} end

		local data = self:GetFile(path)
		local integers = {}
		local last_y = 0
		local last_x = 0

		local function is_lua(str)
			if str:find("return%s", nil) then return true end

			if str:find("local%s", nil) then return true end

			if str:find("%s=%s", nil) then return true end

			if str:find(";", nil, true) then return true end

			return false
		end

		local stringx = require("nattlua.other.string")

		local function process_token(token)
			local type, modifiers = get_semantic_type(token)

			if not type then return end

			assert(tokenTypeMap[type], "invalid type " .. type)
			local pos_data = data.code:SubPosToLineChar(token.start, token.stop)
			local modifier_result = 0

			if modifiers then
				for _, mod in ipairs(modifiers) do
					assert(tokenModifiersMap[mod], "invalid modifier " .. mod)
					modifier_result = bit.bor(modifier_result, bit.lshift(1, tokenModifiersMap[mod]))
				end
			end

			local lines = stringx.split(token:GetValueString(), "\n")
			local line_start = pos_data.line_start - 1
			local line_stop = pos_data.line_stop - 1
			local char_start = pos_data.character_start - 1
			local char_stop = pos_data.character_stop - 1
			--
			local start_line = table.remove(lines, 1)
			local middle_lines

			if #lines > 1 then middle_lines = lines end

			if start_line then
				local len = #start_line
				local y = line_start - last_y
				local x = char_start - last_x

				if y ~= 0 then x = char_start end

				if x >= 0 and y >= 0 then
					table.insert(integers, y)
					table.insert(integers, x)
					table.insert(integers, len)
					table.insert(integers, tokenTypeMap[type])
					table.insert(integers, modifier_result)
					last_y = line_start
					last_x = char_start
				end
			end

			if middle_lines then
				for i, line in ipairs(middle_lines) do
					local len = #line
					local y = line_start - last_y + i
					local x = 0

					if x >= 0 and y >= 0 then
						table.insert(integers, y)
						table.insert(integers, x)
						table.insert(integers, len)
						table.insert(integers, tokenTypeMap[type])
						table.insert(integers, modifier_result)
						last_y = last_y + 1
						last_x = 0
					end
				end
			end
		end

		for i, token in ipairs(data.tokens) do
			if token:HasWhitespace() then
				for _, token in ipairs(token:GetWhitespace()) do
					process_token(token)
				end
			end

			local prev = data.tokens[i - 1]
			local types = token:FindType()

			if
				token.type == "string" and
				(
					(
						data.tokens[i - 1].sub_type == "loadstring" or
						data.tokens[i - 2].sub_type == "loadstring"
					)
					or
					(
						data.tokens[i - 1].sub_type == "cdef" or
						data.tokens[i - 2].sub_type == "cdef"
					)
					or
					types[1] and
					types[1].Type == "string" and
					types[1].lua_compiler or
					is_lua(token:GetValueString())
				)
			then
				local func_kind = data.tokens[i - 1]:GetValueString() or data.tokens[i - 2]:GetValueString()
				local str, start = token:DecomposeString()
				local tokens
				local offset = token.start + #start - 1

				if types[1] and types[1].c_tokens then
					tokens = types[1].c_tokens
					local new_tokens = {}

					for i, token in ipairs(tokens) do
						if token:ValueEquals("TYPEOF_CDECL") then
							local offset = tokens[i + 2].stop

							for i = i + 3, #tokens - 3 do
								local token = tokens[i]
								token.start = token.start - offset
								token.stop = token.stop - offset
								table.insert(new_tokens, token)
							end

							break
						end
					end

					if new_tokens[1] then tokens = new_tokens end
				elseif types[1] and types[1].lua_compiler then
					tokens = types[1].lua_compiler.Tokens
				elseif is_probably_lua(str) then
					local compiler = Compiler(str, "temp")
					compiler.OnDiagnostic = function() end
					local ok, err = compiler:Lex()

					if ok then tokens = compiler.Tokens end
				end

				if tokens then
					process_token(Token.NewVirtualToken("fake", start, token.start, token.start + #start))

					for i, t in ipairs(tokens) do
						local t2 = Token.NewVirtualToken(t.type, t:GetValueString(), t.start + offset, t.stop + offset)

						if t:HasWhitespace() then
							for _, ws in ipairs(t:GetWhitespace()) do
								process_token(
									Token.NewVirtualToken(ws.type, ws:GetValueString(), ws.start + offset, ws.stop + offset)
								)
							end
						end

						process_token(t2)
					end

					process_token(Token.NewVirtualToken("fake", start, token.stop, token.stop + #start))
				else
					process_token(token)
				end
			else
				process_token(token)
			end
		end

		return integers
	end
end

return META