fix(files): remove string tree-sitter parser (#1041)

* fix(files): remove string tree-sitter parser

With the change on nightly neovim/neovim#35988,
string parser creates a scratch buffer under the hood, and uses that as
a source. This drastically decreases the performance of the current
code.

This commit refactors the code to remove the string parser, and adds own
loading of the org buffers for the tree-sitter usage.
Performance should be same as before, with the downside that all of the
org files that are in org_agenda_files are now loaded as unlisted
buffers.

* ci(tests): Run tests only on version 0.11.3+

For some reason, tests on older versions fail without reporting any
actual issues in the tests. Skip running tests on those versions.

Author: kristijanhusak

.github/workflows/tests.yml

@@ -29,9 +29,6 @@ jobs:
           - windows-latest
           - macos-latest
         version:
-          - v0.11.0
-          - v0.11.1
-          - v0.11.2
           - v0.11.3
           - v0.11.4
           - nightly

lua/orgmode/colors/highlighter/markup/init.lua

@@ -169,7 +169,7 @@ end
 ---@return OrgMarkupPreparedHighlight[]
 function OrgMarkup:get_prepared_headline_highlights(headline)
   local highlights =
-    self:get_node_highlights(headline:node(), headline.file:get_source(), select(1, headline:node():range()))
+    self:get_node_highlights(headline:node(), headline.file:bufnr(), select(1, headline:node():range()))
 
   local result = {}
 

lua/orgmode/files/file.lua

@@ -17,14 +17,13 @@ local Memoize = require('orgmode.utils.memoize')
 
 ---@class OrgFileOpts
 ---@field filename string
----@field lines string[]
----@field bufnr? number
+---@field buf number
 
 ---@class OrgFile
 ---@field filename string
+---@field buf number
 ---@field index number
 ---@field lines string[]
----@field content string
 ---@field metadata OrgFileMetadata
 ---@field parser vim.treesitter.LanguageTree
 ---@field root TSNode
@@ -44,15 +43,17 @@ function OrgFile:new(opts)
   local stat = vim.uv.fs_stat(opts.filename)
   local data = {
     filename = opts.filename,
-    lines = opts.lines,
-    content = table.concat(opts.lines, '\n'),
     index = 0,
+    buf = opts.buf or -1,
     metadata = {
       mtime = stat and stat.mtime.nsec or 0,
       mtime_sec = stat and stat.mtime.sec or 0,
-      changedtick = opts.bufnr and vim.api.nvim_buf_get_changedtick(opts.bufnr) or 0,
+      changedtick = opts.buf and vim.api.nvim_buf_get_changedtick(opts.buf) or 0,
     },
   }
+  if data.buf > 0 then
+    data.lines = self:_get_lines(data.buf)
+  end
   setmetatable(data, self)
   return data
 end
@@ -62,28 +63,29 @@ end
 function OrgFile.load(filename)
   local bufnr = utils.get_buffer_by_filename(filename)
 
-  if
-    bufnr > -1
-    and vim.api.nvim_buf_is_loaded(bufnr)
-    and vim.api.nvim_get_option_value('filetype', { buf = bufnr }) == 'org'
-  then
+  if bufnr > -1 and vim.api.nvim_buf_is_loaded(bufnr) and vim.bo[bufnr].filetype == 'org' then
     return Promise.resolve(OrgFile:new({
       filename = filename,
-      lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false),
-      bufnr = bufnr,
+      buf = bufnr,
     }))
   end
 
   if not vim.uv.fs_stat(filename) or not utils.is_org_file(filename) then
     return Promise.resolve(false)
   end
 
-  return utils.readfile(filename, { schedule = true }):next(function(lines)
-    return OrgFile:new({
-      filename = filename,
-      lines = lines,
-    })
-  end)
+  bufnr = vim.fn.bufadd(filename)
+
+  if bufnr == 0 then
+    return Promise.resolve(false)
+  end
+
+  vim.fn.bufload(bufnr)
+
+  return Promise.resolve(OrgFile:new({
+    filename = filename,
+    buf = bufnr,
+  }))
 end
 
 ---Reload the file if it has been modified
@@ -94,15 +96,34 @@ function OrgFile:reload()
   end
 
   local bufnr = self:bufnr()
+  local buf_changed = false
+  local file_changed = false
 
-  if bufnr > -1 then
-    local updated_file = self:_update_lines(vim.api.nvim_buf_get_lines(bufnr, 0, -1, false), bufnr)
-    return Promise.resolve(updated_file)
+  if bufnr then
+    local new_changedtick = vim.api.nvim_buf_get_changedtick(bufnr)
+    buf_changed = self.metadata.changedtick ~= new_changedtick
+    self.metadata.changedtick = new_changedtick
+    if buf_changed then
+      self.lines = self:_get_lines(bufnr)
+    end
   end
-
-  return utils.readfile(self.filename, { schedule = true }):next(function(lines)
-    return self:_update_lines(lines)
-  end)
+  local stat = vim.uv.fs_stat(self.filename)
+  if stat then
+    local new_mtime_nsec = stat.mtime.nsec
+    local new_mtime_sec = stat.mtime.sec
+    file_changed = (new_mtime_nsec > 0 and self.metadata.mtime ~= new_mtime_nsec)
+      or self.metadata.mtime_sec ~= new_mtime_sec
+    self.metadata.mtime = new_mtime_nsec
+    self.metadata.mtime_sec = new_mtime_sec
+  end
+
+  if file_changed and not buf_changed then
+    return utils.readfile(self.filename, { schedule = true }):next(function(lines)
+      self.lines = lines
+      return self
+    end)
+  end
+  return Promise.resolve(self)
 end
 
 ---sync reload the file if it has been modified
@@ -146,7 +167,9 @@ function OrgFile:is_modified()
   local bufnr = self:bufnr()
   if bufnr > -1 then
     local cur_changedtick = vim.api.nvim_buf_get_changedtick(bufnr)
-    return cur_changedtick ~= self.metadata.changedtick
+    if cur_changedtick ~= self.metadata.changedtick then
+      return true
+    end
   end
   local stat = vim.uv.fs_stat(self.filename)
   if not stat then
@@ -166,7 +189,7 @@ function OrgFile:parse(skip_if_not_modified)
   if skip_if_not_modified and self.root and not self:is_modified() then
     return self.root
   end
-  self.parser = self:_get_parser()
+  self.parser = ts.get_parser(self:bufnr(), 'org', {})
   local trees = self.parser:parse()
   self.root = trees[1]:root()
   return self.root
@@ -185,7 +208,7 @@ function OrgFile:get_ts_matches(query, parent_node)
   local ts_query = ts_utils.get_query(query)
   local matches = {}
 
-  for _, match, _ in ts_query:iter_matches(parent_node, self:get_source(), nil, nil, { all = true }) do
+  for _, match, _ in ts_query:iter_matches(parent_node, self:bufnr(), nil, nil, { all = true }) do
     local items = {}
     for id, nodes in pairs(match) do
       local name = ts_query.captures[id]
@@ -215,7 +238,7 @@ function OrgFile:get_ts_captures(query, node)
   local ts_query = ts_utils.get_query(query)
   local matches = {}
 
-  for _, match in ts_query:iter_captures(node, self:get_source()) do
+  for _, match in ts_query:iter_captures(node, self:bufnr()) do
     table.insert(matches, match)
   end
   return matches
@@ -466,13 +489,13 @@ function OrgFile:get_node_text(node, range)
     return ''
   end
   if range then
-    return ts.get_node_text(node, self:get_source(), {
+    return ts.get_node_text(node, self:bufnr(), {
       metadata = {
         range = range,
       },
     })
   end
-  return ts.get_node_text(node, self:get_source())
+  return ts.get_node_text(node, self:bufnr())
 end
 
 ---@param node? TSNode
@@ -534,19 +557,22 @@ end
 
 ---@return number
 function OrgFile:bufnr()
-  local bufnr = utils.get_buffer_by_filename(self.filename)
+  local bufnr = self.buf
   -- Do not consider unloaded buffers as valid
   -- Treesitter is not working in them
   if bufnr > -1 and vim.api.nvim_buf_is_loaded(bufnr) then
     return bufnr
   end
-  return -1
+  local new_bufnr = vim.fn.bufadd(self.filename)
+  vim.fn.bufload(new_bufnr)
+  self.buf = new_bufnr
+  return new_bufnr
 end
 
 ---Return valid buffer handle or throw an error if it's not valid
 ---@return number
 function OrgFile:get_valid_bufnr()
-  local bufnr = utils.get_buffer_by_filename(self.filename)
+  local bufnr = self:bufnr()
   if bufnr < 0 then
     error('[orgmode] No valid buffer for file ' .. self.filename .. ' to edit', 0)
   end
@@ -784,7 +810,7 @@ function OrgFile:get_links()
     (link_desc) @link
   ]])
 
-  local source = self:get_source()
+  local source = self:bufnr()
   for _, node in ipairs(matches) do
     table.insert(links, Hyperlink.from_node(node, source))
   end
@@ -805,7 +831,7 @@ function OrgFile:get_footnote_references()
 
   local footnotes = {}
   local processed_lines = {}
-  for _, match in ts_query:iter_captures(self.root, self:get_source()) do
+  for _, match in ts_query:iter_captures(self.root, self:bufnr()) do
     local line_start, _, line_end = match:range()
     if not processed_lines[line_start] then
       if line_start == line_end then
@@ -895,51 +921,14 @@ function OrgFile:_get_directive(directive_name)
 end
 
 ---@private
----@param lines string[]
----@param bufnr? number
-function OrgFile:_update_lines(lines, bufnr)
-  self.lines = lines
-  self.content = table.concat(lines, '\n')
-  self:parse()
-  if bufnr then
-    self.metadata.changedtick = vim.api.nvim_buf_get_changedtick(bufnr)
-  end
-  local stat = vim.uv.fs_stat(self.filename)
-  if stat then
-    self.metadata.mtime = stat.mtime.nsec
-    self.metadata.mtime_sec = stat.mtime.sec
-  end
-  return self
-end
-
----@private
----@return vim.treesitter.LanguageTree
-function OrgFile:_get_parser()
-  local bufnr = self:bufnr()
-
-  if bufnr > -1 then
-    -- Always get the fresh parser for the buffer
-    return ts.get_parser(bufnr, 'org', {})
-  end
-
-  -- In case the buffer got unloaded, go back to string parser
-  if not self.parser or self:is_modified() or type(self.parser:source()) == 'number' then
-    return ts.get_string_parser(self.content, 'org', {})
-  end
-
-  return self.parser
-end
-
---- Get the ts source for the file
---- If there is a buffer, return buffer number
---- Otherwise, return the string content
----@return integer | string
-function OrgFile:get_source()
-  local bufnr = self:bufnr()
-  if bufnr > -1 then
-    return bufnr
+---Get all buffer lines, ensure empty buffer returns empty table
+---@return string[]
+function OrgFile:_get_lines(bufnr)
+  local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false)
+  if #lines == 1 and lines[1] == '' then
+    lines = {}
   end
-  return self.content
+  return lines
 end
 
 return OrgFile

lua/orgmode/files/headline.lua

@@ -738,7 +738,7 @@ function Headline:get_plan_dates()
     if name ~= 'NONE' then
       has_plan_dates = true
     end
-    dates[name:upper()] = Date.from_node(timestamp, self.file:get_source(), {
+    dates[name:upper()] = Date.from_node(timestamp, self.file:bufnr(), {
       type = name:upper(),
     })
     dates_nodes[name:upper()] = node
@@ -792,7 +792,7 @@ function Headline:get_non_plan_dates()
   end
 
   local all_dates = {}
-  local source = self.file:get_source()
+  local source = self.file:bufnr()
   for _, match in ipairs(matches) do
     local dates = Date.from_node(match, source)
     vim.list_extend(all_dates, dates)

lua/orgmode/init.lua

@@ -91,6 +91,15 @@ end
 
 function Org:setup_autocmds()
   local org_augroup = vim.api.nvim_create_augroup('orgmode_nvim', { clear = true })
+  vim.api.nvim_create_autocmd('BufWinEnter', {
+    pattern = { '*.org', '*.org_archive' },
+    group = org_augroup,
+    callback = function(event)
+      if not vim.bo[event.buf].filetype or vim.bo[event.buf].filetype == '' then
+        vim.bo[event.buf].filetype = 'org'
+      end
+    end,
+  })
   vim.api.nvim_create_autocmd('BufWritePost', {
     pattern = { '*.org', '*.org_archive' },
     group = org_augroup,

tests/plenary/agenda/agenda_item_spec.lua

@@ -8,7 +8,7 @@ local helpers = require('tests.plenary.helpers')
 local function generate(content_line, keyword)
   keyword = keyword or 'TODO'
 
-  local file = helpers.create_file_instance({
+  local file = helpers.create_file({
     '* ' .. keyword .. ' This is some content',
     content_line,
   })

tests/plenary/capture/capture_spec.lua

@@ -198,7 +198,7 @@ describe('Refile', function()
     })
 
     local capture_lines = { '** baz' }
-    local capture_file = helpers.create_file_instance(capture_lines)
+    local capture_file = helpers.create_file(capture_lines)
     local item = capture_file:get_headlines()[1]
 
     ---@diagnostic disable-next-line: invisible
@@ -226,7 +226,7 @@ describe('Capture', function()
     local destination_file = helpers.create_file({})
 
     local capture_lines = { '* foo' }
-    local capture_file = helpers.create_file_instance(capture_lines)
+    local capture_file = helpers.create_file(capture_lines)
     local item = capture_file:get_headlines()[1]
     local template = Template:new({
       properties = {
@@ -310,7 +310,7 @@ describe('Capture', function()
     })
 
     local capture_lines = { '** baz' }
-    local capture_file = helpers.create_file_instance(capture_lines)
+    local capture_file = helpers.create_file(capture_lines)
     local item = capture_file:get_headlines()[1]
     local template = Template:new({
       properties = {
@@ -355,7 +355,7 @@ describe('Capture', function()
     })
 
     local capture_lines = { '** baz' }
-    local capture_file = helpers.create_file_instance(capture_lines)
+    local capture_file = helpers.create_file(capture_lines)
     local item = capture_file:get_headlines()[1]
     local template = Template:new({
       regexp = 'appendhere',