perf(blame): some improvements

- Ensure ':Gitsigns blame' utilizes the blame cache.
- Rewrite the blame runner to process output incrementally.
- Make the blame cache more efficient.
- Move the blame processing code to a separate module.

doc/gitsigns.txt

@@ -308,8 +308,6 @@ blame_line({opts}, {callback?})                        *gitsigns.blame_line()*
                              Display full commit message with hunk.
                            • {ignore_whitespace}: (boolean)
                              Ignore whitespace when running blame.
-                           • {rev}: (string)
-                             Revision to blame against.
                            • {extra_opts}: (string[])
                              Extra options passed to `git-blame`.
 

lua/gitsigns/actions.lua

@@ -977,8 +977,6 @@ end
 ---       Display full commit message with hunk.
 ---     • {ignore_whitespace}: (boolean)
 ---       Ignore whitespace when running blame.
----     • {rev}: (string)
----       Revision to blame against.
 ---     • {extra_opts}: (string[])
 ---       Extra options passed to `git-blame`.
 M.blame_line = async.create(1, function(opts)

lua/gitsigns/blame.lua

@@ -220,11 +220,8 @@ M.blame = function()
     return
   end
 
-  local blame = bcache:run_blame(nil, { rev = bcache.git_obj.revision })
-  if not blame then
-    dprint('No blame info')
-    return
-  end
+  bcache:get_blame()
+  local blame = assert(bcache.blame)
 
   -- Save position to align 'scrollbind'
   local top = vim.fn.line('w0') + vim.wo.scrolloff

lua/gitsigns/cache.lua

@@ -53,6 +53,7 @@ local sleep = async.wrap(2, function(duration, cb)
   vim.defer_fn(cb, duration)
 end)
 
+--- @async
 --- @private
 function CacheEntry:wait_for_hunks()
   local loop_protect = 0
@@ -65,71 +66,61 @@ end
 -- If a file contains has up to this amount of lines, then
 -- always blame the whole file, otherwise only blame one line
 -- at a time.
-local BLAME_THRESHOLD_LEN = 1000000
+local BLAME_THRESHOLD_LEN = 10000
 
 --- @async
+--- @private
 --- @param lnum? integer
---- @param opts Gitsigns.BlameOpts
---- @return table<integer,Gitsigns.BlameInfo?>?
+--- @param opts? Gitsigns.BlameOpts
+--- @return table<integer,Gitsigns.BlameInfo?>
+--- @return boolean? full
 function CacheEntry:run_blame(lnum, opts)
   local bufnr = self.bufnr
-  local blame_cache --- @type table<integer,Gitsigns.BlameInfo?>?
+  local blame --- @type table<integer,Gitsigns.BlameInfo?>?
+  local lnum0 --- @type integer?
   repeat
     local buftext = util.buf_lines(bufnr, true)
     local tick = vim.b[bufnr].changedtick
-    local lnum0 = #buftext > BLAME_THRESHOLD_LEN and lnum or nil
+    lnum0 = #buftext > BLAME_THRESHOLD_LEN and lnum or nil
     -- TODO(lewis6991): Cancel blame on changedtick
-    blame_cache = self.git_obj:run_blame(buftext, lnum0, opts)
+    blame = self.git_obj:run_blame(buftext, lnum0, self.git_obj.revision, opts)
     async.scheduler()
     if not vim.api.nvim_buf_is_valid(bufnr) then
-      return
+      return {}
     end
   until vim.b[bufnr].changedtick == tick
-  return blame_cache
+  return blame, lnum0 == nil
 end
 
---- @param file string
---- @param lnum integer
---- @return Gitsigns.BlameInfo
-local function get_blame_nc(file, lnum)
-  local Git = require('gitsigns.git')
-
-  return {
-    orig_lnum = 0,
-    final_lnum = lnum,
-    commit = Git.not_committed(file),
-    filename = file,
-  }
-end
-
---- @param lnum integer
---- @param opts Gitsigns.BlameOpts
+--- If lnum is nil then run blame for the entire buffer.
+--- @async
+--- @param lnum? integer
+--- @param opts? Gitsigns.BlameOpts
 --- @return Gitsigns.BlameInfo?
 function CacheEntry:get_blame(lnum, opts)
-  if opts.rev then
-    local buftext = util.buf_lines(self.bufnr)
-    return self.git_obj:run_blame(buftext, lnum, opts)[lnum]
-  end
+  local blame = self.blame
 
-  local blame_cache = self.blame
-
-  if not blame_cache or not blame_cache[lnum] then
+  if not blame or (lnum and not blame[lnum]) then
     self:wait_for_hunks()
+    blame = blame or {}
     local Hunks = require('gitsigns.hunks')
-    if Hunks.find_hunk(lnum, self.hunks) then
+    if lnum and Hunks.find_hunk(lnum, self.hunks) then
       --- Bypass running blame (which can be expensive) if we know lnum is in a hunk
-      blame_cache = blame_cache or {}
-      blame_cache[lnum] = get_blame_nc(self.git_obj.relpath, lnum)
+      local Blame = require('gitsigns.git.blame')
+      blame[lnum] = Blame.get_blame_nc(self.git_obj.relpath, lnum)
     else
-      -- Refresh cache
-      blame_cache = self:run_blame(lnum, opts)
+      -- Refresh/update cache
+      local b, full = self:run_blame(lnum, opts)
+      if lnum and not full then
+        blame[lnum] = b[lnum]
+      else
+        blame = b
+      end
     end
-    self.blame = blame_cache
+    self.blame = blame
   end
 
-  if blame_cache then
-    return blame_cache[lnum]
-  end
+  return blame[lnum]
 end
 
 function CacheEntry:destroy()
@@ -139,7 +130,7 @@ function CacheEntry:destroy()
   end
 end
 
----@type table<integer,Gitsigns.CacheEntry>
+---@type table<integer,Gitsigns.CacheEntry?>
 M.cache = {}
 
 --- @param bufnr integer

lua/gitsigns/config.lua

@@ -44,7 +44,6 @@
 
 --- @class (exact) Gitsigns.BlameOpts
 --- @field ignore_whitespace? boolean
---- @field rev? string
 --- @field extra_opts? string[]
 
 --- @class (exact) Gitsigns.LineBlameOpts : Gitsigns.BlameOpts

lua/gitsigns/diffthis.lua

@@ -17,7 +17,7 @@ local M = {}
 --- @param dbufnr integer
 --- @param base string?
 local function bufread(bufnr, dbufnr, base)
-  local bcache = cache[bufnr]
+  local bcache = assert(cache[bufnr])
   base = util.norm_base(base)
   local text --- @type string[]
   if base == bcache.git_obj.revision then
@@ -52,8 +52,9 @@ end
 --- @param bufnr integer
 --- @param dbufnr integer
 --- @param base string?
-local bufwrite = async.create(3, function(bufnr, dbufnr, base)
-  local bcache = cache[bufnr]
+--- @param _callback? fun()
+local bufwrite = async.create(3, function(bufnr, dbufnr, base, _callback)
+  local bcache = assert(cache[bufnr])
   local buftext = util.buf_lines(dbufnr)
   base = util.norm_base(base)
   bcache.git_obj:stage_lines(buftext)
@@ -151,7 +152,8 @@ end
 
 --- @param base string?
 --- @param opts Gitsigns.DiffthisOpts
-M.diffthis = async.create(2, function(base, opts)
+--- @param _callback? fun()
+M.diffthis = async.create(2, function(base, opts, _callback)
   if vim.wo.diff then
     return
   end
@@ -176,7 +178,8 @@ end)
 
 --- @param bufnr integer
 --- @param base string
-M.show = async.create(2, function(bufnr, base)
+--- @param _callback? fun()
+M.show = async.create(2, function(bufnr, base, _callback)
   __FUNC__ = 'show'
   local bufname = create_show_buf(bufnr, base)
   if not bufname then
@@ -205,16 +208,15 @@ end
 
 -- This function needs to be throttled as there is a call to vim.ui.input
 --- @param bufnr integer
-M.update = throttle_by_id(async.create(1, function(bufnr)
+--- @param _callback? fun()
+M.update = throttle_by_id(async.create(1, function(bufnr, _callback)
   if not vim.wo.diff then
     return
   end
 
-  local bcache = cache[bufnr]
-
   -- Note this will be the bufname for the currently set base
   -- which are the only ones we want to update
-  local bufname = bcache:get_rev_bufname()
+  local bufname = assert(cache[bufnr]):get_rev_bufname()
 
   for _, w in ipairs(api.nvim_list_wins()) do
     if api.nvim_win_is_valid(w) then

lua/gitsigns/git.lua

@@ -5,15 +5,8 @@ local log = require('gitsigns.debug.log')
 local util = require('gitsigns.util')
 local system = require('gitsigns.system').system
 
-local gs_config = require('gitsigns.config')
-local config = gs_config.config
-
 local uv = vim.uv or vim.loop
 
-local dprint = log.dprint
-local dprintf = log.dprintf
-local error_once = require('gitsigns.message').error_once
-
 local check_version = require('gitsigns.git.version').check
 
 local M = {}
@@ -516,14 +509,14 @@ end
 --- @return string[] stdout, string? stderr
 function Obj:get_show_text(revision)
   if revision and not self.relpath then
-    dprint('no relpath')
+    log.dprint('no relpath')
     return {}
   end
 
   local object = revision and (revision .. ':' .. self.relpath) or self.object_name
 
   if not object then
-    dprint('no revision or object_name')
+    log.dprint('no revision or object_name')
     return { '' }
   end
 
@@ -584,182 +577,13 @@ end
 --- @field previous_filename? string
 --- @field previous_sha? string
 
-local NOT_COMMITTED = {
-  author = 'Not Committed Yet',
-  author_mail = '<not.committed.yet>',
-  committer = 'Not Committed Yet',
-  committer_mail = '<not.committed.yet>',
-}
-
---- @param file string
---- @return Gitsigns.CommitInfo
-function M.not_committed(file)
-  local time = os.time()
-  return {
-    sha = string.rep('0', 40),
-    abbrev_sha = string.rep('0', 8),
-    author = 'Not Committed Yet',
-    author_mail = '<not.committed.yet>',
-    author_tz = '+0000',
-    author_time = time,
-    committer = 'Not Committed Yet',
-    committer_time = time,
-    committer_mail = '<not.committed.yet>',
-    committer_tz = '+0000',
-    summary = 'Version of ' .. file,
-  }
-end
-
----@param x any
----@return integer
-local function asinteger(x)
-  return assert(tonumber(x))
-end
-
 --- @param lines string[]
 --- @param lnum? integer
+--- @param revision? string
 --- @param opts? Gitsigns.BlameOpts
---- @return table<integer,Gitsigns.BlameInfo?>?
-function Obj:run_blame(lines, lnum, opts)
-  local ret = {} --- @type table<integer,Gitsigns.BlameInfo>
-
-  if not self.object_name or self.repo.abbrev_head == '' then
-    -- As we support attaching to untracked files we need to return something if
-    -- the file isn't isn't tracked in git.
-    -- If abbrev_head is empty, then assume the repo has no commits
-    local commit = M.not_committed(self.file)
-    for i in ipairs(lines) do
-      ret[i] = {
-        orig_lnum = 0,
-        final_lnum = i,
-        commit = commit,
-        filename = self.file,
-      }
-    end
-    return ret
-  end
-
-  local args = { 'blame', '--contents', '-', '--incremental' }
-
-  opts = opts or {}
-
-  if opts.ignore_whitespace then
-    args[#args + 1] = '-w'
-  end
-
-  if lnum then
-    vim.list_extend(args, { '-L', lnum .. ',+1' })
-  end
-
-  if opts.extra_opts then
-    vim.list_extend(args, opts.extra_opts)
-  end
-
-  local ignore_file = self.repo.toplevel .. '/.git-blame-ignore-revs'
-  if uv.fs_stat(ignore_file) then
-    vim.list_extend(args, { '--ignore-revs-file', ignore_file })
-  end
-
-  args[#args + 1] = opts.rev
-  args[#args + 1] = '--'
-  args[#args + 1] = self.file
-
-  local results, stderr = self:command(args, { stdin = lines, ignore_error = true })
-  if stderr then
-    error_once('Error running git-blame: ' .. stderr)
-    return
-  end
-
-  if #results == 0 then
-    return
-  end
-
-  local commits = {} --- @type table<string,Gitsigns.CommitInfo>
-  local i = 1
-
-  while i <= #results do
-    --- @param pat? string
-    --- @return string
-    local function get(pat)
-      local l = assert(results[i])
-      i = i + 1
-      if pat then
-        return l:match(pat)
-      end
-      return l
-    end
-
-    local function peek(pat)
-      local l = results[i]
-      if l and pat then
-        return l:match(pat)
-      end
-      return l
-    end
-
-    local sha, orig_lnum_str, final_lnum_str, size_str = get('(%x+) (%d+) (%d+) (%d+)')
-    local orig_lnum = asinteger(orig_lnum_str)
-    local final_lnum = asinteger(final_lnum_str)
-    local size = asinteger(size_str)
-
-    if peek():match('^author ') then
-      --- @type table<string,string|true>
-      local commit = {
-        sha = sha,
-        abbrev_sha = sha:sub(1, 8),
-      }
-
-      -- filename terminates the entry
-      while peek() and not (peek():match('^filename ') or peek():match('^previous ')) do
-        local l = get()
-        local key, value = l:match('^([^%s]+) (.*)')
-        if key then
-          if vim.endswith(key, '_time') then
-            value = tonumber(value)
-          end
-          key = key:gsub('%-', '_') --- @type string
-          commit[key] = value
-        else
-          commit[l] = true
-          if l ~= 'boundary' then
-            dprintf("Unknown tag: '%s'", l)
-          end
-        end
-      end
-
-      -- New in git 2.41:
-      -- The output given by "git blame" that attributes a line to contents
-      -- taken from the file specified by the "--contents" option shows it
-      -- differently from a line attributed to the working tree file.
-      if
-        commit.author_mail == '<external.file>'
-        or commit.author_mail == 'External file (--contents)'
-      then
-        commit = vim.tbl_extend('force', commit, NOT_COMMITTED)
-      end
-      commits[sha] = commit
-    end
-
-    local previous_sha, previous_filename = peek():match('^previous (%x+) (.*)')
-    if previous_sha then
-      get()
-    end
-
-    local filename = assert(get():match('^filename (.*)'))
-
-    for j = 0, size - 1 do
-      ret[final_lnum + j] = {
-        final_lnum = final_lnum + j,
-        orig_lnum = orig_lnum + j,
-        commit = commits[sha],
-        filename = filename,
-        previous_filename = previous_filename,
-        previous_sha = previous_sha,
-      }
-    end
-  end
-
-  return ret
+--- @return table<integer,Gitsigns.BlameInfo?>
+function Obj:run_blame(lines, lnum, revision, opts)
+  return require('gitsigns.git.blame').run_blame(self, lines, lnum, revision, opts)
 end
 
 --- @param obj Gitsigns.GitObj
@@ -858,7 +682,7 @@ end
 --- @return Gitsigns.GitObj?
 function Obj.new(file, revision, encoding, gitdir, toplevel)
   if in_git_dir(file) then
-    dprint('In git dir')
+    log.dprint('In git dir')
     return nil
   end
   local self = setmetatable({}, { __index = Obj })
@@ -873,7 +697,7 @@ function Obj.new(file, revision, encoding, gitdir, toplevel)
   self.repo = Repo.new(util.dirname(file), gitdir, toplevel)
 
   if not self.repo.gitdir then
-    dprint('Not in git repo')
+    log.dprint('Not in git repo')
     return nil
   end
 

lua/gitsigns/git/blame.lua

@@ -0,0 +1,212 @@
+local uv = vim.uv or vim.loop
+
+local error_once = require('gitsigns.message').error_once
+local dprintf = require('gitsigns.debug.log').dprintf
+
+local NOT_COMMITTED = {
+  author = 'Not Committed Yet',
+  author_mail = '<not.committed.yet>',
+  committer = 'Not Committed Yet',
+  committer_mail = '<not.committed.yet>',
+}
+
+local M = {}
+
+--- @param file string
+--- @return Gitsigns.CommitInfo
+local function not_committed(file)
+  local time = os.time()
+  return {
+    sha = string.rep('0', 40),
+    abbrev_sha = string.rep('0', 8),
+    author = 'Not Committed Yet',
+    author_mail = '<not.committed.yet>',
+    author_tz = '+0000',
+    author_time = time,
+    committer = 'Not Committed Yet',
+    committer_time = time,
+    committer_mail = '<not.committed.yet>',
+    committer_tz = '+0000',
+    summary = 'Version of ' .. file,
+  }
+end
+
+--- @param file string
+--- @param lnum integer
+--- @return Gitsigns.BlameInfo
+function M.get_blame_nc(file, lnum)
+  return {
+    orig_lnum = 0,
+    final_lnum = lnum,
+    commit = not_committed(file),
+    filename = file,
+  }
+end
+
+---@param x any
+---@return integer
+local function asinteger(x)
+  return assert(tonumber(x))
+end
+
+--- @param data_lines string[]
+--- @param i integer
+--- @param commits table<string,Gitsigns.CommitInfo>
+--- @param result table<integer,Gitsigns.BlameInfo>
+--- @return integer i
+local function incremental_iter(data_lines, i, commits, result)
+  local line = assert(data_lines[i])
+  i = i + 1
+
+  --- @type string, string, string, string
+  local sha, orig_lnum_str, final_lnum_str, size_str = line:match('(%x+) (%d+) (%d+) (%d+)')
+  if not sha then
+    return i
+  end
+
+  local orig_lnum = asinteger(orig_lnum_str)
+  local final_lnum = asinteger(final_lnum_str)
+  local size = asinteger(size_str)
+
+  --- @type table<string,string|true>
+  local commit = commits[sha] or {
+    sha = sha,
+    abbrev_sha = sha:sub(1, 8),
+  }
+
+  --- @type string, string
+  local previous_sha, previous_filename
+
+  -- filename terminates the entry
+  while data_lines[i] and not data_lines[i]:match('^filename ') do
+    local l = assert(data_lines[i])
+    i = i + 1
+    local key, value = l:match('^([^%s]+) (.*)')
+    if key == 'previous' then
+      previous_sha, previous_filename = data_lines[i]:match('^previous (%x+) (.*)')
+    elseif key then
+      key = key:gsub('%-', '_') --- @type string
+      if vim.endswith(key, '_time') then
+        value = tonumber(value)
+      end
+      commit[key] = value
+    else
+      commit[l] = true
+      if l ~= 'boundary' then
+        dprintf("Unknown tag: '%s'", l)
+      end
+    end
+  end
+
+  local filename = assert(data_lines[i]:match('^filename (.*)'))
+
+  -- New in git 2.41:
+  -- The output given by "git blame" that attributes a line to contents
+  -- taken from the file specified by the "--contents" option shows it
+  -- differently from a line attributed to the working tree file.
+  if
+    commit.author_mail == '<external.file>'
+    or commit.author_mail == 'External file (--contents)'
+  then
+    commit = vim.tbl_extend('force', commit, NOT_COMMITTED)
+  end
+  commits[sha] = commit
+
+  for j = 0, size - 1 do
+    result[final_lnum + j] = {
+      final_lnum = final_lnum + j,
+      orig_lnum = orig_lnum + j,
+      commit = commits[sha],
+      filename = filename,
+      previous_filename = previous_filename,
+      previous_sha = previous_sha,
+    }
+  end
+
+  return i
+end
+
+--- @param data? string
+--- @param commits table<string,Gitsigns.CommitInfo>
+--- @param result table<integer,Gitsigns.BlameInfo>
+local function process_incremental(data, commits, result)
+  if not data then
+    return
+  end
+
+  local data_lines = vim.split(data, '\n')
+  local i = 1
+
+  while i <= #data_lines do
+    i = incremental_iter(data_lines, i, commits, result)
+  end
+end
+
+--- @param obj Gitsigns.GitObj
+--- @param lines string[]
+--- @param lnum? integer
+--- @param revision? string
+--- @param opts? Gitsigns.BlameOpts
+--- @return table<integer, Gitsigns.BlameInfo>
+function M.run_blame(obj, lines, lnum, revision, opts)
+  local ret = {} --- @type table<integer,Gitsigns.BlameInfo>
+
+  if not obj.object_name or obj.repo.abbrev_head == '' then
+    -- As we support attaching to untracked files we need to return something if
+    -- the file isn't isn't tracked in git.
+    -- If abbrev_head is empty, then assume the repo has no commits
+    local commit = not_committed(obj.file)
+    for i in ipairs(lines) do
+      ret[i] = {
+        orig_lnum = 0,
+        final_lnum = i,
+        commit = commit,
+        filename = obj.file,
+      }
+    end
+    return ret
+  end
+
+  local args = { 'blame', '--contents', '-', '--incremental' }
+
+  opts = opts or {}
+
+  if opts.ignore_whitespace then
+    args[#args + 1] = '-w'
+  end
+
+  if lnum then
+    vim.list_extend(args, { '-L', lnum .. ',+1' })
+  end
+
+  if opts.extra_opts then
+    vim.list_extend(args, opts.extra_opts)
+  end
+
+  local ignore_file = obj.repo.toplevel .. '/.git-blame-ignore-revs'
+  if uv.fs_stat(ignore_file) then
+    vim.list_extend(args, { '--ignore-revs-file', ignore_file })
+  end
+
+  args[#args + 1] = revision
+  args[#args + 1] = '--'
+  args[#args + 1] = obj.file
+
+  local commits = {} --- @type table<string,Gitsigns.CommitInfo>
+
+  --- @param data string?
+  local function on_stdout(_, data)
+    process_incremental(data, commits, ret)
+  end
+
+  local _, stderr = obj:command(args, { stdin = lines, stdout = on_stdout, ignore_error = true })
+
+  if stderr then
+    error_once('Error running git-blame: ' .. stderr)
+    return {}
+  end
+
+  return ret
+end
+
+return M