feat!(previewer): replace plenary.filetype with vim.filetype.match (#2529)

README.md

@@ -390,13 +390,15 @@ Built-in functions. Ready to be bound to any key you like.
 The default previewers are from now on `vim_buffer_` previewers. They use vim
 buffers for displaying files and use tree-sitter or regex for file highlighting.
 
-These previewers are guessing the filetype of the selected file, so there might
+These previewers are using `vim.filetype` to guess the filetype for the
-be cases where they miss, leading to wrong highlights. This is because we can't
+selected file. The guessing is done by inspecting the filename, the head of the
-determine the filetype in the traditional way: We don't do `bufload` and instead
+file(shebang) and the tail of the file (modeline). If you have trouble with
-read the file asynchronously with `vim.loop.fs_` and attach only a highlighter;
+filetype detection you should read `:help vim.filetype`.
-otherwise the speed of the previewer would slow down considerably. If you want
+
-to configure more filetypes, take a look at
+We need to do it manually because we can't determine the filetype in the
-[plenary wiki](https://github.com/nvim-lua/plenary.nvim#plenaryfiletype).
+traditional way: We don't do `bufload` and instead read the file asynchronously
+with `vim.loop.fs_` and attach only a highlighter; otherwise the speed of the
+previewer would slow down considerably.
 
 If you want to configure the `vim_buffer_` previewer (e.g. you want the line to wrap), do this:
 

doc/telescope.txt

@@ -442,8 +442,8 @@ telescope.setup({opts})                                    *telescope.setup()*
 
         Fields:
           - check_mime_type:  Use `file` if available to try to infer whether the
-                              file to preview is a binary if plenary's
+                              file to preview is a binary if filetype
-                              filetype detection fails.
+                              detection fails.
                               Windows users get `file` from:
                               https://github.com/julian-r/file-windows
                               Set to false to attempt to preview any mime type.
@@ -496,11 +496,10 @@ telescope.setup({opts})                                    *telescope.setup()*
                                   end,
                                 }
                               The configuration recipes for relevant examples.
-                              Note: if plenary does not recognize your filetype yet --
+                              Note: we use vim.filetype filetype detection,
-                              1) Please consider contributing to:
+                                    so if you have troubles with files not
-                                 $PLENARY_REPO/data/plenary/filetypes/builtin.lua
+                                    highlighting correctly, please read
-                              2) Register your filetype locally as per link
+                                    |vim.filetype|
-                                 https://github.com/nvim-lua/plenary.nvim#plenaryfiletype
                               Default: nil
           - treesitter:       Determines whether the previewer performs treesitter
                               highlighting, which falls back to regex-based highlighting.

doc/telescope_changelog.txt

@@ -268,4 +268,15 @@ branch 0.1.x (or version, currently 0.1.1) which will not receive this version
 bump and will continue to offer support for older Neovim versions.
 
 
+                                                    *telescope.changelog-2529*
+
+Date: June 09, 2023
+PR: https://github.com/nvim-telescope/telescope.nvim/pull/2529
+
+We finally removed usage of `plenary.filetype` to determine filetypes for
+previewing and replaced it with `vim.filetype`. So if you have highlighting
+issues you no longer have to configure `plenary`, but rather read
+|vim.filetype|.
+
+
  vim:tw=78:ts=8:ft=help:norl:

lua/telescope/config.lua

@@ -540,8 +540,8 @@ append(
 
     Fields:
       - check_mime_type:  Use `file` if available to try to infer whether the
-                          file to preview is a binary if plenary's
+                          file to preview is a binary if filetype
-                          filetype detection fails.
+                          detection fails.
                           Windows users get `file` from:
                           https://github.com/julian-r/file-windows
                           Set to false to attempt to preview any mime type.
@@ -594,11 +594,10 @@ append(
                               end,
                             }
                           The configuration recipes for relevant examples.
-                          Note: if plenary does not recognize your filetype yet --
+                          Note: we use vim.filetype filetype detection,
-                          1) Please consider contributing to:
+                                so if you have troubles with files not
-                             $PLENARY_REPO/data/plenary/filetypes/builtin.lua
+                                highlighting correctly, please read
-                          2) Register your filetype locally as per link
+                                |vim.filetype|
-                             https://github.com/nvim-lua/plenary.nvim#plenaryfiletype
                           Default: nil
       - treesitter:       Determines whether the previewer performs treesitter
                           highlighting, which falls back to regex-based highlighting.

lua/telescope/previewers/buffer_previewer.lua

@@ -5,7 +5,6 @@ local putils = require "telescope.previewers.utils"
 local Previewer = require "telescope.previewers.previewer"
 local conf = require("telescope.config").values
 
-local pfiletype = require "plenary.filetype"
 local pscan = require "plenary.scandir"
 
 local buf_delete = utils.buf_delete
@@ -201,7 +200,6 @@ previewers.file_maker = function(filepath, bufnr, opts)
   if opts.use_ft_detect == nil then
     opts.use_ft_detect = true
   end
-  opts.ft = opts.use_ft_detect and pfiletype.detect(filepath)
   if opts.bufname ~= filepath then
     if not vim.in_fast_event() then
       filepath = vim.fn.expand(filepath)
@@ -218,76 +216,74 @@ previewers.file_maker = function(filepath, bufnr, opts)
       if stat.type == "directory" then
         handle_directory_preview(filepath, bufnr, opts)
       else
-        if opts.preview.check_mime_type == true and has_file and opts.ft == "" then
+        vim.schedule(function()
-          -- avoid SIGABRT in buffer previewer happening with utils.get_os_command_output
+          opts.ft = opts.use_ft_detect and putils.filetype_detect(filepath)
-          local output = capture(string.format([[file --mime-type -b "%s"]], filepath))
+          if opts.preview.check_mime_type == true and has_file and (opts.ft == nil or opts.ft == "") then
-          local mime_type = vim.split(output, "/")
+            -- avoid SIGABRT in buffer previewer happening with utils.get_os_command_output
-          if mime_type[1] ~= "text" and mime_type[1] ~= "inode" and mime_type[2] ~= "json" then
+            local output = capture(string.format([[file --mime-type -b "%s"]], filepath))
-            if type(opts.preview.mime_hook) == "function" then
+            local mime_type = vim.split(output, "/")
-              vim.schedule_wrap(opts.preview.mime_hook)(filepath, bufnr, opts)
+            if mime_type[1] ~= "text" and mime_type[1] ~= "inode" and mime_type[2] ~= "json" then
-            else
+              if type(opts.preview.mime_hook) == "function" then
-              vim.schedule_wrap(putils.set_preview_message)(
+                opts.preview.mime_hook(filepath, bufnr, opts)
-                bufnr,
+              else
-                opts.winid,
+                putils.set_preview_message(
-                "Binary cannot be previewed",
+                  bufnr,
-                opts.preview.msg_bg_fillchar
+                  opts.winid,
-              )
+                  "Binary cannot be previewed",
-            end
+                  opts.preview.msg_bg_fillchar
-            return
+                )
-          end
+              end
-          if mime_type[2] == "json" then
-            opts.ft = "json"
-          end
-        end
-
-        if opts.preview.filesize_limit then
-          local mb_filesize = math.floor(stat.size / bytes_to_megabytes)
-          if mb_filesize > opts.preview.filesize_limit then
-            if type(opts.preview.filesize_hook) == "function" then
-              vim.schedule_wrap(opts.preview.filesize_hook)(filepath, bufnr, opts)
-            else
-              vim.schedule_wrap(putils.set_preview_message)(
-                bufnr,
-                opts.winid,
-                "File exceeds preview size limit",
-                opts.preview.msg_bg_fillchar
-              )
-            end
-            return
-          end
-        end
-
-        opts.start_time = vim.loop.hrtime()
-        Path:new(filepath):_read_async(vim.schedule_wrap(function(data)
-          if not vim.api.nvim_buf_is_valid(bufnr) then
-            return
-          end
-          local processed_data = split(data, "[\r]?\n", _, opts)
-
-          if processed_data then
-            local ok = pcall(vim.api.nvim_buf_set_lines, bufnr, 0, -1, false, processed_data)
-            if not ok then
               return
             end
-
+            if mime_type[2] == "json" then
-            if opts.callback then
+              opts.ft = "json"
-              opts.callback(bufnr)
             end
-            putils.highlighter(bufnr, opts.ft, opts)
-          else
-            if type(opts.preview.timeout_hook) == "function" then
-              vim.schedule_wrap(opts.preview.timeout_hook)(filepath, bufnr, opts)
-            else
-              vim.schedule_wrap(putils.set_preview_message)(
-                bufnr,
-                opts.winid,
-                "Previewer timed out",
-                opts.preview.msg_bg_fillchar
-              )
-            end
-            return
           end
-        end))
+
+          if opts.preview.filesize_limit then
+            local mb_filesize = math.floor(stat.size / bytes_to_megabytes)
+            if mb_filesize > opts.preview.filesize_limit then
+              if type(opts.preview.filesize_hook) == "function" then
+                opts.preview.filesize_hook(filepath, bufnr, opts)
+              else
+                putils.set_preview_message(
+                  bufnr,
+                  opts.winid,
+                  "File exceeds preview size limit",
+                  opts.preview.msg_bg_fillchar
+                )
+              end
+              return
+            end
+          end
+
+          opts.start_time = vim.loop.hrtime()
+          Path:new(filepath):_read_async(vim.schedule_wrap(function(data)
+            if not vim.api.nvim_buf_is_valid(bufnr) then
+              return
+            end
+            local processed_data = split(data, "[\r]?\n", _, opts)
+
+            if processed_data then
+              local ok = pcall(vim.api.nvim_buf_set_lines, bufnr, 0, -1, false, processed_data)
+              if not ok then
+                return
+              end
+
+              if opts.callback then
+                opts.callback(bufnr)
+              end
+              putils.highlighter(bufnr, opts.ft, opts)
+            else
+              if type(opts.preview.timeout_hook) == "function" then
+                opts.preview.timeout_hook(filepath, bufnr, opts)
+              else
+                putils.set_preview_message(bufnr, opts.winid, "Previewer timed out", opts.preview.msg_bg_fillchar)
+              end
+              return
+            end
+          end))
+        end)
       end
     end)
   else
@@ -431,7 +427,7 @@ previewers.new_buffer_previewer = function(opts)
             data = {
               title = entry.preview_title,
               bufname = self.state.bufname,
-              filetype = pfiletype.detect(self.state.bufname or ""),
+              filetype = putils.filetype_detect(self.state.bufname or ""),
             },
           })
         end)
@@ -851,7 +847,7 @@ previewers.git_commit_diff_as_was = defaulter(function(opts)
       local cmd = { "git", "--no-pager", "show" }
       local cf = opts.current_file and Path:new(opts.current_file):make_relative(opts.cwd)
       local value = cf and (entry.value .. ":" .. cf) or entry.value
-      local ft = cf and pfiletype.detect(value) or "diff"
+      local ft = cf and putils.filetype_detect(value) or "diff"
       table.insert(cmd, value)
 
       putils.job_maker(cmd, self.state.bufnr, {

lua/telescope/previewers/utils.lua

@@ -4,9 +4,55 @@ local strings = require "plenary.strings"
 local conf = require("telescope.config").values
 
 local Job = require "plenary.job"
+local Path = require "plenary.path"
 
 local utils = {}
 
+local detect_from_shebang = function(p)
+  local s = p:readbyterange(0, 256)
+  if not s then
+    local lines = vim.split(s, "\n")
+    return vim.filetype.match { contents = lines }
+  end
+end
+
+local parse_modeline = function(tail)
+  if tail:find "vim:" then
+    return tail:match ".*:ft=([^: ]*):.*$" or ""
+  end
+end
+
+local detect_from_modeline = function(p)
+  local s = p:readbyterange(-256, 256)
+  if s then
+    local lines = vim.split(s, "\n")
+    local idx = lines[#lines] ~= "" and #lines or #lines - 1
+    if idx >= 1 then
+      return parse_modeline(lines[idx])
+    end
+  end
+end
+
+utils.filetype_detect = function(filepath)
+  local match = vim.filetype.match { filename = filepath }
+  if match and match ~= "" then
+    return match
+  end
+
+  local p = Path:new(filepath)
+  if p and p:exists() then
+    match = detect_from_shebang(p)
+    if match and match ~= "" then
+      return match
+    end
+
+    match = detect_from_modeline(p)
+    if match and match ~= "" then
+      return match
+    end
+  end
+end
+
 utils.with_preview_window = function(status, bufnr, callable)
   if bufnr and vim.api.nvim_buf_call and false then
     vim.api.nvim_buf_call(bufnr, callable)