feat: vimgrep entry maker using rg --json

lua/telescope/algos/base64.lua

@@ -0,0 +1,190 @@
+--[[
+
+ base64 -- v1.5.3 public domain Lua base64 encoder/decoder
+ no warranty implied; use at your own risk
+
+ Needs bit32.extract function. If not present it's implemented using BitOp
+ or Lua 5.3 native bit operators. For Lua 5.1 fallbacks to pure Lua
+ implementation inspired by Rici Lake's post:
+   http://ricilake.blogspot.co.uk/2007/10/iterating-bits-in-lua.html
+
+ author: Ilya Kolbin (iskolbin@gmail.com)
+ url: github.com/iskolbin/lbase64
+
+ COMPATIBILITY
+
+ Lua 5.1+, LuaJIT
+
+ LICENSE
+
+ See end of file for license information.
+
+--]]
+
+local base64 = {}
+
+local extract = _G.bit32 and _G.bit32.extract -- Lua 5.2/Lua 5.3 in compatibility mode
+if not extract then
+  if _G.bit then -- LuaJIT
+    local shl, shr, band = _G.bit.lshift, _G.bit.rshift, _G.bit.band
+    extract = function(v, from, width)
+      return band(shr(v, from), shl(1, width) - 1)
+    end
+  elseif _G._VERSION == "Lua 5.1" then
+    extract = function(v, from, width)
+      local w = 0
+      local flag = 2 ^ from
+      for i = 0, width - 1 do
+        local flag2 = flag + flag
+        if v % flag2 >= flag then
+          w = w + 2 ^ i
+        end
+        flag = flag2
+      end
+      return w
+    end
+  else -- Lua 5.3+
+    extract = load [[return function( v, from, width )
+			return ( v >> from ) & ((1 << width) - 1)
+		end]]()
+  end
+end
+
+function base64.makeencoder(s62, s63, spad)
+  local encoder = {}
+  -- stylua: ignore
+	for b64code, char in pairs{[0]='A','B','C','D','E','F','G','H','I','J',
+		'K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y',
+		'Z','a','b','c','d','e','f','g','h','i','j','k','l','m','n',
+		'o','p','q','r','s','t','u','v','w','x','y','z','0','1','2',
+		'3','4','5','6','7','8','9',s62 or '+',s63 or'/',spad or'='} do
+		encoder[b64code] = char:byte()
+	end
+  return encoder
+end
+
+function base64.makedecoder(s62, s63, spad)
+  local decoder = {}
+  for b64code, charcode in pairs(base64.makeencoder(s62, s63, spad)) do
+    decoder[charcode] = b64code
+  end
+  return decoder
+end
+
+local DEFAULT_ENCODER = base64.makeencoder()
+local DEFAULT_DECODER = base64.makedecoder()
+
+local char, concat = string.char, table.concat
+
+function base64.encode(str, encoder, usecaching)
+  encoder = encoder or DEFAULT_ENCODER
+  local t, k, n = {}, 1, #str
+  local lastn = n % 3
+  local cache = {}
+  for i = 1, n - lastn, 3 do
+    local a, b, c = str:byte(i, i + 2)
+    local v = a * 0x10000 + b * 0x100 + c
+    local s
+    if usecaching then
+      s = cache[v]
+      if not s then
+        s = char(
+          encoder[extract(v, 18, 6)],
+          encoder[extract(v, 12, 6)],
+          encoder[extract(v, 6, 6)],
+          encoder[extract(v, 0, 6)]
+        )
+        cache[v] = s
+      end
+    else
+      s = char(
+        encoder[extract(v, 18, 6)],
+        encoder[extract(v, 12, 6)],
+        encoder[extract(v, 6, 6)],
+        encoder[extract(v, 0, 6)]
+      )
+    end
+    t[k] = s
+    k = k + 1
+  end
+  if lastn == 2 then
+    local a, b = str:byte(n - 1, n)
+    local v = a * 0x10000 + b * 0x100
+    t[k] = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[extract(v, 6, 6)], encoder[64])
+  elseif lastn == 1 then
+    local v = str:byte(n) * 0x10000
+    t[k] = char(encoder[extract(v, 18, 6)], encoder[extract(v, 12, 6)], encoder[64], encoder[64])
+  end
+  return concat(t)
+end
+
+function base64.decode(b64, decoder, usecaching)
+  decoder = decoder or DEFAULT_DECODER
+  local pattern = "[^%w%+%/%=]"
+  if decoder then
+    local s62, s63
+    for charcode, b64code in pairs(decoder) do
+      if b64code == 62 then
+        s62 = charcode
+      elseif b64code == 63 then
+        s63 = charcode
+      end
+    end
+    pattern = ("[^%%w%%%s%%%s%%=]"):format(char(s62), char(s63))
+  end
+  b64 = b64:gsub(pattern, "")
+  local cache = usecaching and {}
+  local t, k = {}, 1
+  local n = #b64
+  local padding = b64:sub(-2) == "==" and 2 or b64:sub(-1) == "=" and 1 or 0
+  for i = 1, padding > 0 and n - 4 or n, 4 do
+    local a, b, c, d = b64:byte(i, i + 3)
+    local s
+    if usecaching then
+      local v0 = a * 0x1000000 + b * 0x10000 + c * 0x100 + d
+      s = cache[v0]
+      if not s then
+        local v = decoder[a] * 0x40000 + decoder[b] * 0x1000 + decoder[c] * 0x40 + decoder[d]
+        s = char(extract(v, 16, 8), extract(v, 8, 8), extract(v, 0, 8))
+        cache[v0] = s
+      end
+    else
+      local v = decoder[a] * 0x40000 + decoder[b] * 0x1000 + decoder[c] * 0x40 + decoder[d]
+      s = char(extract(v, 16, 8), extract(v, 8, 8), extract(v, 0, 8))
+    end
+    t[k] = s
+    k = k + 1
+  end
+  if padding == 1 then
+    local a, b, c = b64:byte(n - 3, n - 1)
+    local v = decoder[a] * 0x40000 + decoder[b] * 0x1000 + decoder[c] * 0x40
+    t[k] = char(extract(v, 16, 8), extract(v, 8, 8))
+  elseif padding == 2 then
+    local a, b = b64:byte(n - 3, n - 2)
+    local v = decoder[a] * 0x40000 + decoder[b] * 0x1000
+    t[k] = char(extract(v, 16, 8))
+  end
+  return concat(t)
+end
+
+return base64
+
+--[[
+MIT LICENSE
+Copyright (c) 2018 Ilya Kolbin
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+--]]

lua/telescope/builtin/__files.lua

@@ -59,51 +59,10 @@ local get_open_filelist = function(grep_open_files, cwd)
   return filelist
 end
 
-local opts_contain_invert = function(args)
-  local invert = false
-  local files_with_matches = false
-
-  for _, v in ipairs(args) do
-    if v == "--invert-match" then
-      invert = true
-    elseif v == "--files-with-matches" or v == "--files-without-match" then
-      files_with_matches = true
-    end
-
-    if #v >= 2 and v:sub(1, 1) == "-" and v:sub(2, 2) ~= "-" then
-      local non_option = false
-      for i = 2, #v do
-        local vi = v:sub(i, i)
-        if vi == "=" then -- ignore option -g=xxx
-          break
-        elseif vi == "g" or vi == "f" or vi == "m" or vi == "e" or vi == "r" or vi == "t" or vi == "T" then
-          non_option = true
-        elseif non_option == false and vi == "v" then
-          invert = true
-        elseif non_option == false and vi == "l" then
-          files_with_matches = true
-        end
-      end
-    end
-  end
-  return invert, files_with_matches
-end
-
--- Special keys:
---  opts.search_dirs -- list of directory to search in
---  opts.grep_open_files -- boolean to restrict search to open files
-files.live_grep = function(opts)
+---@param opts table: picker options
+---@return table, table: rg options and search paths
+local get_vimgrep_args = function(opts)
   local vimgrep_arguments = opts.vimgrep_arguments or conf.vimgrep_arguments
-  local search_dirs = opts.search_dirs
-  local grep_open_files = opts.grep_open_files
-  opts.cwd = opts.cwd and vim.fn.expand(opts.cwd) or vim.loop.cwd()
-
-  local filelist = get_open_filelist(grep_open_files, opts.cwd)
-  if search_dirs then
-    for i, path in ipairs(search_dirs) do
-      search_dirs[i] = vim.fn.expand(path)
-    end
-  end
 
   local additional_args = {}
   if opts.additional_args ~= nil then
@@ -114,49 +73,94 @@ files.live_grep = function(opts)
     end
   end
 
-  if opts.type_filter then
-    additional_args[#additional_args + 1] = "--type=" .. opts.type_filter
-  end
-
   if type(opts.glob_pattern) == "string" then
-    additional_args[#additional_args + 1] = "--glob=" .. opts.glob_pattern
+    table.insert(additional_args, "--glob=" .. opts.glob_pattern)
   elseif type(opts.glob_pattern) == "table" then
     for i = 1, #opts.glob_pattern do
-      additional_args[#additional_args + 1] = "--glob=" .. opts.glob_pattern[i]
+      table.insert(additional_args, "--glob=" .. opts.glob_pattern[i])
     end
   end
 
+  if opts.type_filter then
+    table.insert(additional_args, "--type=" .. opts.type_filter)
+  end
   if opts.file_encoding then
-    additional_args[#additional_args + 1] = "--encoding=" .. opts.file_encoding
+    table.insert(additional_args, "--encoding=" .. opts.file_encoding)
   end
 
-  local args = flatten { vimgrep_arguments, additional_args }
-  opts.__inverted, opts.__matches = opts_contain_invert(args)
+  local search_paths = {}
+  local open_files = get_open_filelist(opts.grep_open_files, opts.cwd)
+  if open_files then
+    search_paths = open_files
+  elseif opts.search_dirs then
+    for _, path in ipairs(opts.search_dirs) do
+      table.insert(search_paths, vim.fn.expand(path))
+    end
+  end
+
+  return flatten { vimgrep_arguments, additional_args }, search_paths
+end
+
+---@param rg_opts table: list of rg options
+---@return table, boolean: rg options and whether `--json` was applied
+local add_json_opt = function(rg_opts)
+  local JSON_UNSUPPORT_FLAGS = {
+    USE_FILE_ENTRY_MAKER = { "-l", "--files-with-matches", "--files-without-match" },
+    IGNORE = { "--files", "--count", "--count-matches", "-c" },
+  }
+
+  local json_arg = { enabled = false, index = -1 }
+  for idx, arg in ipairs(rg_opts) do
+    if vim.tbl_contains(JSON_UNSUPPORT_FLAGS.USE_FILE_ENTRY_MAKER, arg) then
+      if json_arg.enabled then
+        table.remove(rg_opts, json_arg.index)
+      end
+      return rg_opts, false
+    elseif vim.tbl_contains(JSON_UNSUPPORT_FLAGS.IGNORE, arg) then
+      utils.notify("add_json_opt", {
+        msg = string.format(
+          "%s not supported by grep_string or live_grep - ignoring",
+          table.concat(JSON_UNSUPPORT_FLAGS.IGNORE, ", ")
+        ),
+        level = "WARN",
+      })
+      table.remove(rg_opts, idx)
+    elseif arg == "--json" then
+      json_arg.enabled = true
+      json_arg.index = idx
+    end
+  end
+
+  if not json_arg.enabled then
+    table.insert(rg_opts, "--json")
+  end
+  return rg_opts, true
+end
+
+files.live_grep = function(opts)
+  opts.__finder = "live_grep"
+  opts.cwd = opts.cwd and vim.fn.expand(opts.cwd) or vim.loop.cwd()
+
+  local rg_opts, search_paths = get_vimgrep_args(opts)
+
+  local json_enabled
+  rg_opts, json_enabled = add_json_opt(rg_opts)
+  local entry_maker = json_enabled and make_entry.gen_from_vimgrep_json(opts) or make_entry.gen_from_file(opts)
 
   local live_grepper = finders.new_job(function(prompt)
     if not prompt or prompt == "" then
       return nil
     end
 
-    local search_list = {}
-
-    if grep_open_files then
-      search_list = filelist
-    elseif search_dirs then
-      search_list = search_dirs
-    end
-
-    return flatten { args, "--", prompt, search_list }
-  end, opts.entry_maker or make_entry.gen_from_vimgrep(opts), opts.max_results, opts.cwd)
+    return flatten { rg_opts, "--", prompt, search_paths }
+  end, opts.entry_maker or entry_maker, opts.max_results, opts.cwd)
 
   pickers
     .new(opts, {
       prompt_title = "Live Grep",
       finder = live_grepper,
       previewer = conf.grep_previewer(opts),
-      -- TODO: It would be cool to use `--json` output for this
-      -- and then we could get the highlight positions directly.
-      sorter = sorters.highlighter_only(opts),
+      sorter = sorters.empty(),
       attach_mappings = function(_, map)
         map("i", "<c-space>", actions.to_fuzzy_refine)
         return true
@@ -166,8 +170,9 @@ files.live_grep = function(opts)
 end
 
 files.grep_string = function(opts)
+  opts.__finder = "grep_string"
   opts.cwd = opts.cwd and vim.fn.expand(opts.cwd) or vim.loop.cwd()
-  local vimgrep_arguments = vim.F.if_nil(opts.vimgrep_arguments, conf.vimgrep_arguments)
+  local rg_opts, search_paths = get_vimgrep_args(opts)
   local word
   local visual = vim.fn.mode() == "v"
 
@@ -180,60 +185,26 @@ files.grep_string = function(opts)
   else
     word = vim.F.if_nil(opts.search, vim.fn.expand "<cword>")
   end
-  local search = opts.use_regex and word or escape_chars(word)
+  local pattern = opts.use_regex and word or escape_chars(word)
 
-  local additional_args = {}
-  if opts.additional_args ~= nil then
-    if type(opts.additional_args) == "function" then
-      additional_args = opts.additional_args(opts)
-    elseif type(opts.additional_args) == "table" then
-      additional_args = opts.additional_args
-    end
+  if visual then
+    table.insert(rg_opts, opts.word_match)
   end
 
-  if opts.file_encoding then
-    additional_args[#additional_args + 1] = "--encoding=" .. opts.file_encoding
+  if pattern == "" then
+    table.insert(rg_opts, "-v")
+    pattern = "^[[:space:]]*$"
   end
 
-  if search == "" then
-    search = { "-v", "--", "^[[:space:]]*$" }
-  else
-    search = { "--", search }
-  end
+  local json_enabled
+  rg_opts, json_enabled = add_json_opt(rg_opts)
+  local entry_maker = json_enabled and make_entry.gen_from_vimgrep_json(opts) or make_entry.gen_from_file(opts)
+  opts.entry_maker = opts.entry_maker or entry_maker
 
-  local args
-  if visual == true then
-    args = flatten {
-      vimgrep_arguments,
-      additional_args,
-      search,
-    }
-  else
-    args = flatten {
-      vimgrep_arguments,
-      additional_args,
-      opts.word_match,
-      search,
-    }
-  end
-
-  opts.__inverted, opts.__matches = opts_contain_invert(args)
-
-  if opts.grep_open_files then
-    for _, file in ipairs(get_open_filelist(opts.grep_open_files, opts.cwd)) do
-      table.insert(args, file)
-    end
-  elseif opts.search_dirs then
-    for _, path in ipairs(opts.search_dirs) do
-      table.insert(args, vim.fn.expand(path))
-    end
-  end
-
-  opts.entry_maker = opts.entry_maker or make_entry.gen_from_vimgrep(opts)
   pickers
     .new(opts, {
       prompt_title = "Find Word (" .. word:gsub("\n", "\\n") .. ")",
-      finder = finders.new_oneshot_job(args, opts),
+      finder = finders.new_oneshot_job(flatten { rg_opts, "--", pattern, search_paths }, opts),
       previewer = conf.grep_previewer(opts),
       sorter = conf.generic_sorter(opts),
     })

lua/telescope/builtin/init.lua

@@ -52,8 +52,9 @@ end
 ---@field search_dirs table: directory/directories/files to search, mutually exclusive with `grep_open_files`
 ---@field glob_pattern string|table: argument to be used with `--glob`, e.g. "*.toml", can use the opposite "!*.toml"
 ---@field type_filter string: argument to be used with `--type`, e.g. "rust", see `rg --type-list`
+---@field use_regex boolean: if true, special characters won't be escaped, allows for using regex (default: false)
+---@field word_match string: can be set to `-w` to enable exact word matches
 ---@field additional_args function|table: additional arguments to be passed on. Can be fn(opts) -> tbl
----@field max_results number: define a upper result value
 ---@field disable_coordinates boolean: don't show the line & row numbers (default: false)
 ---@field file_encoding string: file encoding for the entry & previewer
 builtin.live_grep = require_on_exported_call("telescope.builtin.__files").live_grep
@@ -64,11 +65,13 @@ builtin.live_grep = require_on_exported_call("telescope.builtin.__files").live_g
 ---@field search string: the query to search
 ---@field grep_open_files boolean: if true, restrict search to open files only, mutually exclusive with `search_dirs`
 ---@field search_dirs table: directory/directories/files to search, mutually exclusive with `grep_open_files`
+---@field glob_pattern string|table: argument to be used with `--glob`, e.g. "*.toml", can use the opposite "!*.toml"
+---@field type_filter string: argument to be used with `--type`, e.g. "rust", see `rg --type-list`
+---@field additional_args function|table: additional arguments to be passed on. Can be fn(opts) -> tbl
 ---@field use_regex boolean: if true, special characters won't be escaped, allows for using regex (default: false)
 ---@field word_match string: can be set to `-w` to enable exact word matches
----@field additional_args function|table: additional arguments to be passed on. Can be fn(opts) -> tbl
----@field disable_coordinates boolean: don't show the line and row numbers (default: false)
 ---@field only_sort_text boolean: only sort the text, not the file, line or row (default: false)
+---@field disable_coordinates boolean: don't show the line and row numbers (default: false)
 ---@field file_encoding string: file encoding for the entry & previewer
 builtin.grep_string = require_on_exported_call("telescope.builtin.__files").grep_string
 

lua/telescope/config.lua

@@ -626,24 +626,15 @@ append(
 
 append(
   "vimgrep_arguments",
-  { "rg", "--color=never", "--no-heading", "--with-filename", "--line-number", "--column", "--smart-case" },
+  { "rg", "--smart-case", "--json" },
   [[
     Defines the command that will be used for `live_grep` and `grep_string`
     pickers.
-    Hint: Make sure that color is currently set to `never` because we do
-    not yet interpret color codes
-    Hint 2: Make sure that these options are in your changes arguments:
-      "--no-heading", "--with-filename", "--line-number", "--column"
-    because we need them so the ripgrep output is in the correct format.
 
     Default: {
       "rg",
-      "--color=never",
-      "--no-heading",
-      "--with-filename",
-      "--line-number",
-      "--column",
-      "--smart-case"
+      "--smart-case",
+      "--json",
     }]]
 )
 

lua/telescope/make_entry.lua

@@ -369,6 +369,117 @@ do
   end
 end
 
+function make_entry.gen_from_vimgrep_json(opts)
+  opts = opts or {}
+  local cwd = vim.fn.expand(opts.cwd or vim.loop.cwd() or "")
+
+  local function bytes_or_text_to_str(data)
+    if data.bytes ~= nil then
+      return require("telescope.algos.base64").decode(data.bytes)
+    else
+      return data.text
+    end
+  end
+
+  local mt_vimgrep_entry = {}
+  mt_vimgrep_entry.display = function(entry)
+    local display_filename = utils.transform_path(opts, entry.filename)
+
+    local coordinates = ":"
+    if not opts.disable_coordinates then
+      if entry.lnum then
+        if entry.col then
+          coordinates = string.format(":%s:%s:", entry.lnum, entry.col)
+        else
+          coordinates = string.format(":%s:", entry.lnum)
+        end
+      end
+    end
+
+    local file_pos, hl_group, icon = utils.transform_devicons(
+      entry.filename,
+      string.format("%s%s", display_filename, coordinates),
+      opts.disable_devicons
+    )
+    local file_pos_len = strings.strdisplaywidth(file_pos)
+
+    local displayer = entry_display.create {
+      separator = "",
+      items = {
+        { width = file_pos_len },
+        { remaining = true },
+      },
+    }
+
+    return displayer {
+      {
+        file_pos,
+        function()
+          return {
+            { { 0, #icon }, hl_group },
+            { { #icon, file_pos_len }, "" },
+          }
+        end,
+      },
+      {
+        entry.text,
+        function()
+          if opts.__finder == "grep_string" then
+            return {}
+          end
+
+          local highlights = {}
+          for _, submatch in ipairs(entry.submatches) do
+            table.insert(highlights, { { submatch["start"], submatch["end"] }, "TelescopeMatching" })
+          end
+          return highlights
+        end,
+      },
+    }
+  end
+
+  mt_vimgrep_entry.__index = function(t, k)
+    local override = handle_entry_index(opts, t, k)
+    if override then
+      return override
+    end
+
+    local raw = rawget(mt_vimgrep_entry, k)
+    if raw then
+      return raw
+    end
+
+    if k == "path" then
+      return Path:new({ cwd, t.filename }):absolute()
+    end
+
+    if k == "text" then
+      return t.value
+    end
+
+    if k == "ordinal" then
+      local text = t.text
+      return opts.only_sort_text and text or text .. " " .. t.filename
+    end
+  end
+
+  return function(line)
+    local msg = vim.json.decode(line)
+    if msg == nil or msg.type ~= "match" then
+      return
+    end
+
+    local text = bytes_or_text_to_str(msg.data.lines):gsub("%s+$", "")
+    return setmetatable({
+      value = text,
+      filename = bytes_or_text_to_str(msg.data.path),
+      lnum = msg.data.line_number,
+      col = #msg.data.submatches ~= 0 and msg.data.submatches[1].start + 1 or nil,
+      submatches = msg.data.submatches,
+    }, mt_vimgrep_entry)
+  end
+end
+
 function make_entry.gen_from_git_stash(opts)
   local displayer = entry_display.create {
     separator = " ",