[WIP]: Tue 15 Sep 2020 09:40:20 PM EDT

lua/telescope/pickers.lua

@@ -304,6 +304,9 @@ function Picker:find()
   local selection_strategy = self.selection_strategy or 'reset'
 
   local on_lines = function(_, _, _, first_line, last_line)
+    self._requests_in_flight = 0
+    self._requests_id = vim.loop.hrtime()
+
     if not vim.api.nvim_buf_is_valid(prompt_bufnr) then
       return
     end
@@ -318,6 +321,8 @@ function Picker:find()
     self.manager = pickers.entry_manager(
       self.max_results,
       vim.schedule_wrap(function(index, entry)
+        -- TODO: Should assert some things about request id
+
         local row = self:get_row(index)
 
         -- If it's less than 0, then we don't need to show it at all.
@@ -353,6 +358,8 @@ function Picker:find()
           display = display:gsub("\n", " | ")
           vim.api.nvim_buf_set_lines(results_bufnr, row, row + 1, false, {display})
         end
+
+        self._requests_in_flight = self._requests_in_flight - 1
       end
     ))
 
@@ -364,14 +371,23 @@ function Picker:find()
 
       log.trace("Processing result... ", entry)
 
+      self._requests_in_flight = self._requests_in_flight + 1
+
       if sorter then
-        require('telescope.sorters.multi_thread').score_entry(prompt, entry, self)
+        vim.schedule(function()
+          require('telescope.sorters.multi_thread').score_entry(self._requests_id, prompt, entry, self)
+        end)
       else
         self.manager:add_entry(0, entry)
       end
     end
 
     local process_complete = vim.schedule_wrap(function()
+      vim.wait(100, function()
+        -- print("ID:", self._requests_id, "IN FLIGHT:", self._requests_in_flight)
+        return self._requests_in_flight <= 0
+      end)
+
       -- TODO: We should either: always leave one result or make sure we actually clean up the results when nothing matches
 
       if selection_strategy == 'row' then

lua/telescope/sorters/multi_thread.lua

@@ -2,29 +2,57 @@ local log = require('telescope.log')
 
 local M = {}
 
-function M.score_entry(prompt, entry, picker)
-  local worker = vim.loop.new_work(function(path, prompt, entry)
-    package.path = path
+local current_request_id = nil
 
-    if not FuzzySorter then
-      FuzzySorter = require('telescope.sorters').get_fuzzy_file()
-    end
+local request_id_to_picker = setmetatable({}, {
+  __mode = 'kv'
+})
 
-    -- return pcall(FuzzySorter.score, FuzzySorter, prompt, entry)
-    return true, 3
-  end, vim.schedule_wrap(function(score_ok, sort_score)
-    -- TODO: we should totally make sure that this picker is still doing stuff...
-    -- it could otherwise be done.
-    if not score_ok or sort_score == -1 then
-      log.warn("Sorting failed with:", prompt, entry, sort_score)
-      return
-    end
+local max_entry_id = 0
+local entry_id_to_entry = {}
 
-    -- picker.manager:add_entry(sort_score, entry)
-    print(score_ok, sort_score)
-  end))
+local worker_func = function(path, bound_request_id, entry_id, prompt, entry)
+  package.path = path
 
-  worker:queue(package.path, prompt, type(entry) == "string" and entry or entry.ordinal)
+  if not FuzzySorter then
+    FuzzySorter = require('telescope.sorters').get_fuzzy_file()
+  end
+
+  return bound_request_id, entry_id, pcall(FuzzySorter.score, FuzzySorter, prompt, entry)
+end
+
+local after_func = function(bound_request_id, entry_id, score_ok, sort_score)
+  local picker = request_id_to_picker[bound_request_id]
+
+  if picker._requests_id ~= bound_request_id or bound_request_id ~= current_request_id then
+    return
+  end
+
+  -- TODO: we should totally make sure that this picker is still doing stuff...
+  -- it could otherwise be done.
+  if not score_ok or sort_score == -1 then
+    picker._requests_in_flight = picker._requests_in_flight - 1
+    return
+  end
+
+  local entry = entry_id_to_entry[entry_id]
+  entry_id_to_entry[entry_id] = nil
+
+  picker.manager:add_entry(sort_score, entry)
+end
+
+local worker = vim.loop.new_work(worker_func, after_func)
+
+
+function M.score_entry(bound_request_id, prompt, entry, picker)
+  current_request_id = bound_request_id
+
+  request_id_to_picker[bound_request_id] = picker
+
+  max_entry_id = max_entry_id + 1
+  entry_id_to_entry[max_entry_id] = entry
+
+  worker:queue(package.path, bound_request_id, max_entry_id, prompt, type(entry) == "string" and entry or entry.ordinal)
 end
 
 return M

lua/tests/manual/large_search.lua

@@ -1,16 +1,18 @@
 RELOAD('plenary')
 RELOAD('telescope')
 
+require('telescope')
+
 local finders = require('telescope.finders')
 local make_entry = require('telescope.make_entry')
 local previewers = require('telescope.previewers')
 local pickers = require('telescope.pickers')
 local sorters = require('telescope.sorters')
 
-PERF_DEBUG = 182
+PERF_DEBUG = 455
 vim.api.nvim_buf_set_lines(PERF_DEBUG, 0, -1, false, {})
 
-local cwd = vim.fn.expand("~/build/neovim")
+local cwd = vim.fn.expand("~/plugins/telescope.nvim")
 
 pickers.new {
   prompt = 'Large search',
@@ -29,9 +31,9 @@ pickers.new {
 
 
 COMPLETED = false
--- vim.wait(3000, function()
+-- print(vim.wait(3000, function()
 --   vim.cmd [[redraw!]]
 --   return COMPLETED
--- end, 100)
+-- end, 100))
 -- vim.cmd [[bd!]]
 -- vim.cmd [[stopinsert]]

scratch/threaded_lua.lua

@@ -1,3 +1,15 @@
+--[[
+
+Some open questions:
+
+1. Can I pass a function as a string and deserialize on the other side?
+2. How expensive is said serialization?
+
+
+--]]
+
+
+
 local uv = vim.loop
 
 ThreadsAvailable = {}