Manual merge of into .

data/core/commands/core.lua

@@ -6,10 +6,12 @@ local LogView = require "core.logview"
 
 
 local fullscreen = false
+local restore_title_view = false
 
 local function suggest_directory(text)
   text = common.home_expand(text)
-  return common.home_encode_list(text == "" and core.recent_projects or common.dir_path_suggest(text))
+  return common.home_encode_list((text == "" or text == common.home_expand(common.dirname(core.project_dir))) 
+    and core.recent_projects or common.dir_path_suggest(text))
 end
 
 command.add(nil, {
@@ -27,9 +29,12 @@ command.add(nil, {
 
   ["core:toggle-fullscreen"] = function()
     fullscreen = not fullscreen
+    if fullscreen then
+      restore_title_view = core.title_view.visible
+    end
     system.set_window_mode(fullscreen and "fullscreen" or "normal")
-    core.show_title_bar(not fullscreen)
-    core.title_view:configure_hit_test(not fullscreen)
+    core.show_title_bar(not fullscreen and restore_title_view)
+    core.title_view:configure_hit_test(not fullscreen and restore_title_view)
   end,
 
   ["core:reload-module"] = function()
@@ -66,8 +71,8 @@ command.add(nil, {
   end,
 
   ["core:find-file"] = function()
-    if core.project_files_limit then
-      return command.perform "core:open-file"
+    if not core.project_files_number() then
+       return command.perform "core:open-file"
     end
     local files = {}
     for dir, item in core.get_project_files() do
@@ -149,7 +154,7 @@ command.add(nil, {
   ["core:change-project-folder"] = function()
     local dirname = common.dirname(core.project_dir)
     if dirname then
-      core.command_view:set_text(common.home_encode(dirname) .. PATHSEP)
+      core.command_view:set_text(common.home_encode(dirname))
     end
     core.command_view:enter("Change Project Folder", function(text, item)
       text = system.absolute_path(common.home_expand(item and item.text or text))
@@ -166,7 +171,7 @@ command.add(nil, {
   ["core:open-project-folder"] = function()
     local dirname = common.dirname(core.project_dir)
     if dirname then
-      core.command_view:set_text(common.home_encode(dirname) .. PATHSEP)
+      core.command_view:set_text(common.home_encode(dirname))
     end
     core.command_view:enter("Open Project", function(text, item)
       text = common.home_expand(item and item.text or text)
@@ -191,8 +196,6 @@ command.add(nil, {
         return
       end
       core.add_project_directory(system.absolute_path(text))
-      -- TODO: add the name of directory to prioritize
-      core.reschedule_project_scan()
     end, suggest_directory)
   end,
 

data/core/commands/findreplace.lua

@@ -15,7 +15,8 @@ local find_regex = config.find_regex or false
 local found_expression
 
 local function doc()
-  return core.active_view:is(DocView) and core.active_view.doc or last_view.doc
+  local is_DocView = core.active_view:is(DocView) and not core.active_view:is(CommandView)
+  return is_DocView and core.active_view.doc or (last_view and last_view.doc)
 end
 
 local function get_find_tooltip()
@@ -117,7 +118,7 @@ local function has_selection()
 end
 
 local function has_unique_selection()
-  if not core.active_view:is(DocView) then return false end
+  if not doc() then return false end
   local text = nil
   for idx, line1, col1, line2, col2 in doc():get_selections(true, true) do
     if line1 == line2 and col1 == col2 then return false end
@@ -142,7 +143,7 @@ local function is_in_any_selection(line, col)
   return false
 end
 
-local function select_next(all)
+local function select_add_next(all)
   local il1, ic1 = doc():get_selection(true)
   for idx, l1, c1, l2, c2 in doc():get_selections(true, true) do
     local text = doc():get_text(l1, c1, l2, c2)
@@ -161,21 +162,28 @@ local function select_next(all)
   end
 end
 
-command.add(has_unique_selection, {
-  ["find-replace:select-next"] = function()
-    local l1, c1, l2, c2 = doc():get_selection(true)
-    local text = doc():get_text(l1, c1, l2, c2)
+local function select_next(reverse)
+  local l1, c1, l2, c2 = doc():get_selection(true)
+  local text = doc():get_text(l1, c1, l2, c2)
+  if reverse then
+    l1, c1, l2, c2 = search.find(doc(), l1, c1, text, { wrap = true, reverse = true })
+  else
     l1, c1, l2, c2 = search.find(doc(), l2, c2, text, { wrap = true })
-    if l2 then doc():set_selection(l2, c2, l1, c1) end
-  end,
-  ["find-replace:select-add-next"] = function() select_next(false) end,
-  ["find-replace:select-add-all"] = function() select_next(true) end
+  end
+  if l2 then doc():set_selection(l2, c2, l1, c1) end
+end
+
+command.add(has_unique_selection, {
+  ["find-replace:select-next"] = select_next,
+  ["find-replace:select-previous"] = function() select_next(true) end,
+  ["find-replace:select-add-next"] = select_add_next,
+  ["find-replace:select-add-all"] = function() select_add_next(true) end
 })
 
 command.add("core.docview", {
   ["find-replace:find"] = function()
-    find("Find Text", function(doc, line, col, text, case_sensitive, find_regex)
-      local opt = { wrap = true, no_case = not case_sensitive, regex = find_regex }
+    find("Find Text", function(doc, line, col, text, case_sensitive, find_regex, find_reverse)
+      local opt = { wrap = true, no_case = not case_sensitive, regex = find_regex, reverse = find_reverse }
       return search.find(doc, line, col, text, opt)
     end)
   end,
@@ -221,29 +229,29 @@ command.add(valid_for_finding, {
       core.error("No find to continue from")
     else
       local sl1, sc1, sl2, sc2 = doc():get_selection(true)
-      local line1, col1, line2, col2 = last_fn(doc(), sl1, sc2, last_text, case_sensitive, find_regex)
+      local line1, col1, line2, col2 = last_fn(doc(), sl1, sc2, last_text, case_sensitive, find_regex, false)
       if line1 then
-        if last_view.doc ~= doc() then
-          last_finds = {}
-        end
-        if #last_finds >= max_last_finds then
-          table.remove(last_finds, 1)
-        end
-        table.insert(last_finds, { sl1, sc1, sl2, sc2 })
         doc():set_selection(line2, col2, line1, col1)
         last_view:scroll_to_line(line2, true)
+      else
+        core.error("Couldn't find %q", last_text)
       end
     end
   end,
 
   ["find-replace:previous-find"] = function()
-    local sel = table.remove(last_finds)
-    if not sel or doc() ~= last_view.doc then
-      core.error("No previous finds")
-      return
+    if not last_fn then
+      core.error("No find to continue from")
+    else
+      local sl1, sc1, sl2, sc2 = doc():get_selection(true)
+      local line1, col1, line2, col2 = last_fn(doc(), sl1, sc1, last_text, case_sensitive, find_regex, true)
+      if line1 then
+        doc():set_selection(line2, col2, line1, col1)
+        last_view:scroll_to_line(line2, true)
+      else
+        core.error("Couldn't find %q", last_text)
+      end
     end
-    doc():set_selection(table.unpack(sel))
-    last_view:scroll_to_line(sel[3], true)
   end,
 })
 

data/core/commands/root.lua

@@ -113,7 +113,8 @@ for _, dir in ipairs { "left", "right", "up", "down" } do
       y = node.position.y + (dir == "up"   and -1 or node.size.y + style.divider_size)
     end
     local node = core.root_view.root_node:get_child_overlapping_point(x, y)
-    if not node:get_locked_size() then
+    local sx, sy = node:get_locked_size()
+    if not sx and not sy then
       core.set_active_view(node.active_view)
     end
   end
@@ -121,7 +122,8 @@ end
 
 command.add(function()
   local node = core.root_view:get_active_node()
-  return not node:get_locked_size()
+  local sx, sy = node:get_locked_size()
+  return not sx and not sy
 end, t)
 
 command.add(nil, {

data/core/common.lua

@@ -1,8 +1,8 @@
 local common = {}
 
 
-function common.is_utf8_cont(char)
-  local byte = char:byte()
+function common.is_utf8_cont(s, offset)
+  local byte = s:byte(offset or 1)
   return byte >= 0x80 and byte < 0xc0
 end
 
@@ -280,24 +280,61 @@ local function split_on_slash(s, sep_pattern)
 end
 
 
-function common.normalize_path(filename)
+-- The filename argument given to the function is supposed to
+-- come from system.absolute_path and as such should be an
+-- absolute path without . or .. elements.
+-- This function exists because on Windows the drive letter returned
+-- by system.absolute_path is sometimes with a lower case and sometimes
+-- with an upper case to we normalize to upper case.
+function common.normalize_volume(filename)
   if not filename then return end
+  if PATHSEP == '\\' then
+    local drive, rem = filename:match('^([a-zA-Z]:\\)(.*)')
+    if drive then
+      return drive:upper() .. rem
+    end
+  end
+  return filename
+end
+
+
+function common.normalize_path(filename)
+  if not filename then return end
+  local volume
   if PATHSEP == '\\' then
     filename = filename:gsub('[/\\]', '\\')
-    local drive, rem = filename:match('^([a-zA-Z])(:.*)')
-    filename = drive and drive:upper() .. rem or filename
+    local drive, rem = filename:match('^([a-zA-Z]:\\)(.*)')
+    if drive then
+      volume, filename = drive:upper(), rem
+    else
+      drive, rem = filename:match('^(\\\\[^\\]+\\[^\\]+\\)(.*)')
+      if drive then
+        volume, filename = drive, rem
+      end
+    end
+  else
+    local relpath = filename:match('^/(.+)')
+    if relpath then
+      volume, filename = "/", relpath
+    end
   end
   local parts = split_on_slash(filename, PATHSEP)
   local accu = {}
   for _, part in ipairs(parts) do
-    if part == '..' and #accu > 0 and accu[#accu] ~= ".." then
-      table.remove(accu)
+    if part == '..' then
+      if #accu > 0 and accu[#accu] ~= ".." then
+        table.remove(accu)
+      elseif volume then
+        error("invalid path " .. volume .. filename)
+      else
+        table.insert(accu, part)
+      end
     elseif part ~= '.' then
       table.insert(accu, part)
     end
   end
   local npath = table.concat(accu, PATHSEP)
-  return npath == "" and PATHSEP or npath
+  return (volume or "") .. (npath == "" and PATHSEP or npath)
 end
 
 

data/core/config.lua

@@ -1,6 +1,5 @@
 local config = {}
 
-config.project_scan_rate = 5
 config.fps = 60
 config.max_log_items = 80
 config.message_timeout = 5
@@ -12,8 +11,8 @@ config.symbol_pattern = "[%a_][%w_]*"
 config.non_word_chars = " \t\n/\\()\"':,.;<>~!@#$%^&*|+=[]{}`?-"
 config.undo_merge_timeout = 0.3
 config.max_undos = 10000
-config.max_tabs = 10
-config.always_show_tabs = false
+config.max_tabs = 8
+config.always_show_tabs = true
 config.highlight_current_line = true
 config.line_height = 1.2
 config.indent_size = 2

data/core/doc/highlighter.lua

@@ -1,4 +1,5 @@
 local core = require "core"
+local common = require "core.common"
 local config = require "core.config"
 local tokenizer = require "core.tokenizer"
 local Object = require "core.object"
@@ -40,6 +41,13 @@ end
 
 function Highlighter:reset()
   self.lines = {}
+  self:soft_reset()
+end
+
+function Highlighter:soft_reset()
+  for i=1,#self.lines do
+    self.lines[i] = false
+  end
   self.first_invalid_line = 1
   self.max_wanted_line = 0
 end
@@ -51,16 +59,16 @@ end
 
 function Highlighter:insert_notify(line, n)
   self:invalidate(line)
+  local blanks = { }
   for i = 1, n do
-    table.insert(self.lines, line, nil)
+    blanks[i] = false
   end
+  common.splice(self.lines, line, 0, blanks)
 end
 
 function Highlighter:remove_notify(line, n)
   self:invalidate(line)
-  for i = 1, n do
-    table.remove(self.lines, line)
-  end
+  common.splice(self.lines, line, n)
 end
 
 

data/core/doc/init.lua

@@ -47,7 +47,7 @@ function Doc:reset_syntax()
   local syn = syntax.get(self.filename or "", header)
   if self.syntax ~= syn then
     self.syntax = syn
-    self.highlighter:reset()
+    self.highlighter:soft_reset()
   end
 end
 
@@ -62,12 +62,15 @@ function Doc:load(filename)
   local fp = assert( io.open(filename, "rb") )
   self:reset()
   self.lines = {}
+  local i = 1
   for line in fp:lines() do
     if line:byte(-1) == 13 then
       line = line:sub(1, -2)
       self.crlf = true
     end
     table.insert(self.lines, line .. "\n")
+    self.highlighter.lines[i] = false
+    i = i + 1
   end
   if #self.lines == 0 then
     table.insert(self.lines, "\n")
@@ -306,6 +309,7 @@ local function pop_undo(self, undo_stack, redo_stack, modified)
     self:raw_remove(line1, col1, line2, col2, redo_stack, cmd.time)
   elseif cmd.type == "selection" then
     self.selections = { table.unpack(cmd) }
+    self:sanitize_selection()
   end
 
   modified = modified or (cmd.type ~= "selection")

data/core/doc/search.lua

@@ -22,37 +22,62 @@ local function init_args(doc, line, col, text, opt)
   return doc, line, col, text, opt
 end
 
+-- This function is needed to uniform the behavior of
+-- `regex:cmatch` and `string.find`.
+local function regex_func(text, re, index, _)
+  local s, e = re:cmatch(text, index)
+  return s, e and e - 1
+end
+
+local function rfind(func, text, pattern, index, plain)
+  local s, e = func(text, pattern, 1, plain)
+  local last_s, last_e
+  if index < 0 then index = #text - index + 1 end
+  while e and e <= index do
+    last_s, last_e = s, e
+    s, e = func(text, pattern, s + 1, plain)
+  end
+  return last_s, last_e
+end
+
 
 function search.find(doc, line, col, text, opt)
   doc, line, col, text, opt = init_args(doc, line, col, text, opt)
-
-  local re
+  local plain = not opt.pattern
+  local pattern = text
+  local search_func = string.find
   if opt.regex then
-    re = regex.compile(text, opt.no_case and "i" or "")
+    pattern = regex.compile(text, opt.no_case and "i" or "")
+    search_func = regex_func
   end
-  for line = line, #doc.lines do
+  local start, finish, step = line, #doc.lines, 1
+  if opt.reverse then
+    start, finish, step = line, 1, -1
+  end
+  for line = start, finish, step do
     local line_text = doc.lines[line]
-    if opt.regex then
-      local s, e = re:cmatch(line_text, col)
-      if s then
-        return line, s, line, e
-      end
-      col = 1
-    else
-      if opt.no_case then
-        line_text = line_text:lower()
-      end
-      local s, e = line_text:find(text, col, true)
-      if s then
-        return line, s, line, e + 1
-      end
-      col = 1
+    if opt.no_case and not opt.regex then
+      line_text = line_text:lower()
     end
+    local s, e
+    if opt.reverse then
+      s, e = rfind(search_func, line_text, pattern, col - 1, plain)
+    else
+      s, e = search_func(line_text, pattern, col, plain)
+    end
+    if s then
+      return line, s, line, e + 1
+    end
+    col = opt.reverse and -1 or 1
   end
 
   if opt.wrap then
-    opt = { no_case = opt.no_case, regex = opt.regex }
-    return search.find(doc, 1, 1, text, opt)
+    opt = { no_case = opt.no_case, regex = opt.regex, reverse = opt.reverse }
+    if opt.reverse then
+      return search.find(doc, #doc.lines, #doc.lines[#doc.lines], text, opt)
+    else
+      return search.find(doc, 1, 1, text, opt)
+    end
   end
 end
 

data/core/docview.lua

@@ -410,7 +410,9 @@ function DocView:draw()
 
   local pos = self.position
   x, y = self:get_line_screen_position(minline)
-  core.push_clip_rect(pos.x + gw, pos.y, self.size.x, self.size.y)
+  -- the clip below ensure we don't write on the gutter region. On the
+  -- right side it is redundant with the Node's clip.
+  core.push_clip_rect(pos.x + gw, pos.y, self.size.x - gw, self.size.y)
   for i = minline, maxline do
     self:draw_line_body(i, x, y)
     y = y + lh

data/core/init.lua

@@ -36,7 +36,7 @@ end
 
 
 local function update_recents_project(action, dir_path_abs)
-  local dirname = common.normalize_path(dir_path_abs)
+  local dirname = common.normalize_volume(dir_path_abs)
   if not dirname then return end
   local recents = core.recent_projects
   local n = #recents
@@ -52,23 +52,13 @@ local function update_recents_project(action, dir_path_abs)
 end
 
 
-function core.reschedule_project_scan()
-  if core.project_scan_thread_id then
-    core.threads[core.project_scan_thread_id].wake = 0
-  end
-end
-
-
 function core.set_project_dir(new_dir, change_project_fn)
   local chdir_ok = pcall(system.chdir, new_dir)
   if chdir_ok then
     if change_project_fn then change_project_fn() end
-    core.project_dir = common.normalize_path(new_dir)
+    core.project_dir = common.normalize_volume(new_dir)
     core.project_directories = {}
     core.add_project_directory(new_dir)
-    core.project_files = {}
-    core.project_files_limit = false
-    core.reschedule_project_scan()
     return true
   end
   return false
@@ -102,6 +92,29 @@ local function compare_file(a, b)
   return a.filename < b.filename
 end
 
+
+-- compute a file's info entry completed with "filename" to be used
+-- in project scan or falsy if it shouldn't appear in the list.
+local function get_project_file_info(root, file)
+  local info = system.get_file_info(root .. file)
+  if info then
+    info.filename = strip_leading_path(file)
+    return (info.size < config.file_size_limit * 1e6 and
+      not common.match_pattern(info.filename, config.ignore_files)
+      and info)
+  end
+end
+
+
+-- Predicate function to inhibit directory recursion in get_directory_files
+-- based on a time limit and the number of files.
+local function timed_max_files_pred(dir, filename, entries_count, t_elapsed)
+  local n_limit = entries_count <= config.max_project_files
+  local t_limit = t_elapsed < 20 / config.fps
+  return n_limit and t_limit and core.project_subdir_is_shown(dir, filename)
+end
+
+
 -- "root" will by an absolute path without trailing '/'
 -- "path" will be a path starting with '/' and without trailing '/'
 --    or the empty string.
@@ -110,34 +123,31 @@ end
 -- When recursing "root" will always be the same, only "path" will change.
 -- Returns a list of file "items". In eash item the "filename" will be the
 -- complete file path relative to "root" *without* the trailing '/'.
-local function get_directory_files(root, path, t, recursive, begin_hook)
+local function get_directory_files(dir, root, path, t, entries_count, recurse_pred, begin_hook)
   if begin_hook then begin_hook() end
-  local size_limit = config.file_size_limit * 10e5
+  local t0 = system.get_time()
   local all = system.list_dir(root .. path) or {}
+  local t_elapsed = system.get_time() - t0
   local dirs, files = {}, {}
 
-  local entries_count = 0
-  local max_entries = config.max_project_files
   for _, file in ipairs(all) do
-    if not common.match_pattern(file, config.ignore_files) then
-      local file = path .. PATHSEP .. file
-      local info = system.get_file_info(root .. file)
-      if info and info.size < size_limit then
-        info.filename = strip_leading_path(file)
-        table.insert(info.type == "dir" and dirs or files, info)
-        entries_count = entries_count + 1
-        if recursive and entries_count > max_entries then return nil, entries_count end
-      end
+    local info = get_project_file_info(root, path .. PATHSEP .. file)
+    if info then
+      table.insert(info.type == "dir" and dirs or files, info)
+      entries_count = entries_count + 1
     end
   end
 
+  local recurse_complete = true
   table.sort(dirs, compare_file)
   for _, f in ipairs(dirs) do
     table.insert(t, f)
-    if recursive and entries_count <= max_entries then
-      local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
-      entries_count = entries_count + subdir_count
-      f.scanned = true
+    if recurse_pred(dir, f.filename, entries_count, t_elapsed) then
+      local _, complete, n = get_directory_files(dir, root, PATHSEP .. f.filename, t, entries_count, recurse_pred, begin_hook)
+      recurse_complete = recurse_complete and complete
+      entries_count = n
+    else
+      recurse_complete = false
     end
   end
 
@@ -146,135 +156,319 @@ local function get_directory_files(root, path, t, recursive, begin_hook)
     table.insert(t, f)
   end
 
-  return t, entries_count
+  return t, recurse_complete, entries_count
 end
 
-local function project_scan_thread()
-  local function diff_files(a, b)
-    if #a ~= #b then return true end
-    for i, v in ipairs(a) do
-      if b[i].filename ~= v.filename
-      or b[i].modified ~= v.modified then
-        return true
-      end
-    end
-  end
 
-  while true do
-    -- get project files and replace previous table if the new table is
-    -- different
-    local i = 1
-    while not core.project_files_limit and i <= #core.project_directories do
-      local dir = core.project_directories[i]
-      local t, entries_count = get_directory_files(dir.name, "", {}, true)
-      if diff_files(dir.files, t) then
-        if entries_count > config.max_project_files then
-          core.project_files_limit = true
-          core.status_view:show_message("!", style.accent,
-            "Too many files in project directory: stopped reading at "..
-            config.max_project_files.." files. For more information see "..
-            "usage.md at github.com/franko/lite-xl."
-            )
-        end
-        dir.files = t
-        core.redraw = true
-      end
-      if dir.name == core.project_dir then
-        core.project_files = dir.files
-      end
-      i = i + 1
+function core.project_subdir_set_show(dir, filename, show)
+  dir.shown_subdir[filename] = show
+  if dir.files_limit and PLATFORM == "Linux" then
+    local fullpath = dir.name .. PATHSEP .. filename
+    local watch_fn = show and system.watch_dir_add or system.watch_dir_rm
+    local success = watch_fn(dir.watch_id, fullpath)
+    if not success then
+      core.log("Internal warning: error calling system.watch_dir_%s", show and "add" or "rm")
     end
-
-    -- wait for next scan
-    coroutine.yield(config.project_scan_rate)
   end
 end
 
 
-function core.is_project_folder(dirname)
-  for _, dir in ipairs(core.project_directories) do
-    if dir.name == dirname then
-      return true
-    end
-  end
-  return false
+function core.project_subdir_is_shown(dir, filename)
+  return not dir.files_limit or dir.shown_subdir[filename]
 end
 
 
-function core.scan_project_folder(dirname, filename)
-  for _, dir in ipairs(core.project_directories) do
-    if dir.name == dirname then
-      for i, file in ipairs(dir.files) do
-        local file = dir.files[i]
-        if file.filename == filename then
-          if file.scanned then return end
-          local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
-          for j, new_file in ipairs(new_files) do
-            table.insert(dir.files, i + j, new_file)
-          end
-          file.scanned = true
-          return
+local function show_max_files_warning(dir)
+  local message = dir.slow_filesystem and
+    "Filesystem is too slow: project files will not be indexed." or
+    "Too many files in project directory: stopped reading at "..
+    config.max_project_files.." files. For more information see "..
+    "usage.md at github.com/franko/lite-xl."
+  core.status_view:show_message("!", style.accent, message)
+end
+
+
+local function file_search(files, info)
+  local filename, type = info.filename, info.type
+  local inf, sup = 1, #files
+  while sup - inf > 8 do
+    local curr = math.floor((inf + sup) / 2)
+    if system.path_compare(filename, type, files[curr].filename, files[curr].type) then
+      sup = curr - 1
+    else
+      inf = curr
+    end
+  end
+  repeat
+    if files[inf].filename == filename then
+      return inf, true
+    end
+    inf = inf + 1
+  until inf > sup or system.path_compare(filename, type, files[inf].filename, files[inf].type)
+  return inf, false
+end
+
+
+local function project_scan_add_entry(dir, fileinfo)
+  local index, match = file_search(dir.files, fileinfo)
+  if not match then
+    table.insert(dir.files, index, fileinfo)
+    dir.is_dirty = true
+  end
+end
+
+
+local function files_info_equal(a, b)
+  return a.filename == b.filename and a.type == b.type
+end
+
+-- for "a" inclusive from i1 + 1 and i1 + n
+local function files_list_match(a, i1, n, b)
+  if n ~= #b then return false end
+  for i = 1, n do
+    if not files_info_equal(a[i1 + i], b[i]) then
+      return false
+    end
+  end
+  return true
+end
+
+-- arguments like for files_list_match
+local function files_list_replace(as, i1, n, bs)
+  local m = #bs
+  local i, j = 1, 1
+  while i <= m or i <= n do
+    local a, b = as[i1 + i], bs[j]
+    if i > n or (j <= m and not files_info_equal(a, b) and
+      not system.path_compare(a.filename, a.type, b.filename, b.type))
+    then
+      table.insert(as, i1 + i, b)
+      i, j, n = i + 1, j + 1, n + 1
+    elseif j > m or system.path_compare(a.filename, a.type, b.filename, b.type) then
+      table.remove(as, i1 + i)
+      n = n - 1
+    else
+      i, j = i + 1, j + 1
+    end
+  end
+end
+
+local function project_subdir_bounds(dir, filename)
+  local index, n = 0, #dir.files
+  for i, file in ipairs(dir.files) do
+    local file = dir.files[i]
+    if file.filename == filename then
+      index, n = i, #dir.files - i
+      for j = 1, #dir.files - i do
+        if not common.path_belongs_to(dir.files[i + j].filename, filename) then
+          n = j - 1
+          break
         end
       end
+      return index, n, file
     end
   end
 end
 
+local function rescan_project_subdir(dir, filename_rooted)
+  local new_files = get_directory_files(dir, dir.name, filename_rooted, {}, 0, core.project_subdir_is_shown, coroutine.yield)
+  local index, n = 0, #dir.files
+  if filename_rooted ~= "" then
+    local filename = strip_leading_path(filename_rooted)
+    index, n = project_subdir_bounds(dir, filename)
+  end
 
-local function find_project_files_co(root, path)
-  local size_limit = config.file_size_limit * 10e5
+  if not files_list_match(dir.files, index, n, new_files) then
+    files_list_replace(dir.files, index, n, new_files)
+    dir.is_dirty = true
+    return true
+  end
+end
+
+
+local function add_dir_scan_thread(dir)
+  core.add_thread(function()
+    while true do
+      local has_changes = rescan_project_subdir(dir, "")
+      if has_changes then
+        core.redraw = true -- we run without an event, from a thread
+      end
+      coroutine.yield(5)
+    end
+  end)
+end
+
+-- Populate a project folder top directory by scanning the filesystem.
+local function scan_project_folder(index)
+  local dir = core.project_directories[index]
+  if PLATFORM == "Linux" then
+    local fstype = system.get_fs_type(dir.name)
+    dir.force_rescan = (fstype == "nfs" or fstype == "fuse")
+  end
+  local t, complete, entries_count = get_directory_files(dir, dir.name, "", {}, 0, timed_max_files_pred)
+  if not complete then
+    dir.slow_filesystem = not complete and (entries_count <= config.max_project_files)
+    dir.files_limit = true
+    if not dir.force_rescan then
+      -- Watch non-recursively on Linux only.
+      -- The reason is recursively watching with dmon on linux
+      -- doesn't work on very large directories.
+      dir.watch_id = system.watch_dir(dir.name, PLATFORM ~= "Linux")
+    end
+    if core.status_view then -- May be not yet initialized.
+      show_max_files_warning(dir)
+    end
+  else
+    if not dir.force_rescan then
+      dir.watch_id = system.watch_dir(dir.name, true)
+    end
+  end
+  dir.files = t
+  if dir.force_rescan then
+    add_dir_scan_thread(dir)
+  else
+    core.dir_rescan_add_job(dir, ".")
+  end
+end
+
+
+function core.add_project_directory(path)
+  -- top directories has a file-like "item" but the item.filename
+  -- will be simply the name of the directory, without its path.
+  -- The field item.topdir will identify it as a top level directory.
+  path = common.normalize_volume(path)
+  local dir = {
+    name = path,
+    item = {filename = common.basename(path), type = "dir", topdir = true},
+    files_limit = false,
+    is_dirty = true,
+    shown_subdir = {},
+  }
+  table.insert(core.project_directories, dir)
+  scan_project_folder(#core.project_directories)
+  if path == core.project_dir then
+    core.project_files = dir.files
+  end
+  core.redraw = true
+end
+
+
+function core.update_project_subdir(dir, filename, expanded)
+  local index, n, file = project_subdir_bounds(dir, filename)
+  if index then
+    local new_files = expanded and get_directory_files(dir, dir.name, PATHSEP .. filename, {}, 0, core.project_subdir_is_shown) or {}
+    files_list_replace(dir.files, index, n, new_files)
+    dir.is_dirty = true
+    return true
+  end
+end
+
+
+-- Find files and directories recursively reading from the filesystem.
+-- Filter files and yields file's directory and info table. This latter
+-- is filled to be like required by project directories "files" list.
+local function find_files_rec(root, path)
   local all = system.list_dir(root .. path) or {}
   for _, file in ipairs(all) do
-    if not common.match_pattern(file, config.ignore_files) then
-      local file = path .. PATHSEP .. file
-      local info = system.get_file_info(root .. file)
-      if info and info.size < size_limit then
-        info.filename = strip_leading_path(file)
-        if info.type == "file" then
-          coroutine.yield(root, info)
-        else
-          find_project_files_co(root, PATHSEP .. info.filename)
-        end
+    local file = path .. PATHSEP .. file
+    local info = system.get_file_info(root .. file)
+    if info then
+      info.filename = strip_leading_path(file)
+      if info.type == "file" then
+        coroutine.yield(root, info)
+      else
+        find_files_rec(root, PATHSEP .. info.filename)
       end
     end
   end
 end
 
 
+-- Iterator function to list all project files
 local function project_files_iter(state)
   local dir = core.project_directories[state.dir_index]
-  state.file_index = state.file_index + 1
-  while dir and state.file_index > #dir.files do
-    state.dir_index = state.dir_index + 1
-    state.file_index = 1
-    dir = core.project_directories[state.dir_index]
+  if state.co then
+    -- We have a coroutine to fetch for files, use the coroutine.
+    -- Used for directories that exceeds the files nuumber limit.
+    local ok, name, file = coroutine.resume(state.co, dir.name, "")
+    if ok and name then
+      return name, file
+    else
+      -- The coroutine terminated, increment file/dir counter to scan
+      -- next project directory.
+      state.co = false
+      state.file_index = 1
+      state.dir_index = state.dir_index + 1
+      dir = core.project_directories[state.dir_index]
+    end
+  else
+    -- Increase file/dir counter
+    state.file_index = state.file_index + 1
+    while dir and state.file_index > #dir.files do
+      state.dir_index = state.dir_index + 1
+      state.file_index = 1
+      dir = core.project_directories[state.dir_index]
+    end
   end
   if not dir then return end
+  if dir.files_limit then
+    -- The current project directory is files limited: create a couroutine
+    -- to read files from the filesystem.
+    state.co = coroutine.create(find_files_rec)
+    return project_files_iter(state)
+  end
   return dir.name, dir.files[state.file_index]
 end
 
 
 function core.get_project_files()
-  if core.project_files_limit then
-    return coroutine.wrap(function()
-      for _, dir in ipairs(core.project_directories) do
-        find_project_files_co(dir.name, "")
-      end
-    end)
-  else
-    local state = { dir_index = 1, file_index = 0 }
-    return project_files_iter, state
-  end
+  local state = { dir_index = 1, file_index = 0 }
+  return project_files_iter, state
 end
 
 
 function core.project_files_number()
-  if not core.project_files_limit then
-    local n = 0
-    for i = 1, #core.project_directories do
-      n = n + #core.project_directories[i].files
+  local n = 0
+  for i = 1, #core.project_directories do
+    if core.project_directories[i].files_limit then return end
+    n = n + #core.project_directories[i].files
+  end
+  return n
+end
+
+
+local function project_dir_by_watch_id(watch_id)
+  for i = 1, #core.project_directories do
+    if core.project_directories[i].watch_id == watch_id then
+      return core.project_directories[i]
     end
-    return n
+  end
+end
+
+
+local function project_scan_remove_file(dir, filepath)
+  local fileinfo = { filename = filepath }
+  for _, filetype in ipairs {"dir", "file"} do
+    fileinfo.type = filetype
+    local index, match = file_search(dir.files, fileinfo)
+    if match then
+      table.remove(dir.files, index)
+      dir.is_dirty = true
+      return
+    end
+  end
+end
+
+
+local function project_scan_add_file(dir, filepath)
+  for fragment in string.gmatch(filepath, "([^/\\]+)") do
+    if common.match_pattern(fragment, config.ignore_files) then
+      return
+    end
+  end
+  local fileinfo = get_project_file_info(dir.name, PATHSEP .. filepath)
+  if fileinfo then
+    project_scan_add_entry(dir, fileinfo)
   end
 end
 
@@ -371,19 +565,6 @@ function core.load_user_directory()
 end
 
 
-function core.add_project_directory(path)
-  -- top directories has a file-like "item" but the item.filename
-  -- will be simply the name of the directory, without its path.
-  -- The field item.topdir will identify it as a top level directory.
-  path = common.normalize_path(path)
-  table.insert(core.project_directories, {
-    name = path,
-    item = {filename = common.basename(path), type = "dir", topdir = true},
-    files = {}
-  })
-end
-
-
 function core.remove_project_directory(path)
   -- skip the fist directory because it is the project's directory
   for i = 2, #core.project_directories do
@@ -422,9 +603,9 @@ function core.init()
   Doc = require "core.doc"
 
   if PATHSEP == '\\' then
-    USERDIR = common.normalize_path(USERDIR)
-    DATADIR = common.normalize_path(DATADIR)
-    EXEDIR  = common.normalize_path(EXEDIR)
+    USERDIR = common.normalize_volume(USERDIR)
+    DATADIR = common.normalize_volume(DATADIR)
+    EXEDIR  = common.normalize_volume(EXEDIR)
   end
 
   do
@@ -509,7 +690,6 @@ function core.init()
   cur_node = cur_node:split("down", core.command_view, {y = true})
   cur_node = cur_node:split("down", core.status_view, {y = true})
 
-  core.project_scan_thread_id = core.add_thread(project_scan_thread)
   command.add_defaults()
   local got_user_error = not core.load_user_directory()
   local plugins_success, plugins_refuse_list = core.load_plugins()
@@ -520,6 +700,12 @@ function core.init()
   end
   local got_project_error = not core.load_project_module()
 
+  -- We assume we have just a single project directory here. Now that StatusView
+  -- is there show max files warning if needed.
+  if core.project_directories[1].files_limit then
+    show_max_files_warning(core.project_directories[1])
+  end
+
   for _, filename in ipairs(files) do
     core.root_view:open_doc(core.open_doc(filename))
   end
@@ -910,6 +1096,84 @@ function core.try(fn, ...)
   return false, err
 end
 
+local scheduled_rescan = {}
+
+function core.has_pending_rescan()
+  for _ in pairs(scheduled_rescan) do
+    return true
+  end
+end
+
+
+function core.dir_rescan_add_job(dir, filepath)
+  local dirpath = filepath:match("^(.+)[/\\].+$")
+  local dirpath_rooted = dirpath and PATHSEP .. dirpath or ""
+  local abs_dirpath = dir.name .. dirpath_rooted
+  if dirpath then
+    -- check if the directory is in the project files list, if not exit
+    local dir_index, dir_match = file_search(dir.files, {filename = dirpath, type = "dir"})
+    -- Note that is dir_match is false dir_index greaten than the last valid index.
+    -- We use dir_index to index dir.files below only if dir_match is true.
+    if not dir_match or not core.project_subdir_is_shown(dir, dir.files[dir_index].filename) then return end
+  end
+  local new_time = system.get_time() + 1
+
+  -- evaluate new rescan request versus existing rescan
+  local remove_list = {}
+  for _, rescan in pairs(scheduled_rescan) do
+    if abs_dirpath == rescan.abs_path or common.path_belongs_to(abs_dirpath, rescan.abs_path) then
+      -- abs_dirpath is a subpath of a scan already ongoing: skip
+      rescan.time_limit = new_time
+      return
+    elseif common.path_belongs_to(rescan.abs_path, abs_dirpath) then
+      -- abs_dirpath already cover this rescan: add to the list of rescan to be removed
+      table.insert(remove_list, rescan.abs_path)
+    end
+  end
+  for _, key_path in ipairs(remove_list) do
+    scheduled_rescan[key_path] = nil
+  end
+
+  scheduled_rescan[abs_dirpath] = {dir = dir, path = dirpath_rooted, abs_path = abs_dirpath, time_limit = new_time}
+  core.add_thread(function()
+    while true do
+      local rescan = scheduled_rescan[abs_dirpath]
+      if not rescan then return end
+      if system.get_time() > rescan.time_limit then
+        local has_changes = rescan_project_subdir(rescan.dir, rescan.path)
+        if has_changes then
+          core.redraw = true -- we run without an event, from a thread
+          rescan.time_limit = new_time
+        else
+          scheduled_rescan[rescan.abs_path] = nil
+          return
+        end
+      end
+      coroutine.yield(0.2)
+    end
+  end)
+end
+
+
+-- no-op but can be overrided by plugins
+function core.on_dirmonitor_modify()
+end
+
+
+function core.on_dir_change(watch_id, action, filepath)
+  local dir = project_dir_by_watch_id(watch_id)
+  if not dir then return end
+  core.dir_rescan_add_job(dir, filepath)
+  if action == "delete" then
+    project_scan_remove_file(dir, filepath)
+  elseif action == "create" then
+    project_scan_add_file(dir, filepath)
+    core.on_dirmonitor_modify(dir, filepath);
+  elseif action == "modify" then
+    core.on_dirmonitor_modify(dir, filepath);
+  end
+end
+
 
 function core.on_event(type, ...)
   local did_keymap = false
@@ -950,6 +1214,8 @@ function core.on_event(type, ...)
     end
   elseif type == "focuslost" then
     core.root_view:on_focus_lost(...)
+  elseif type == "dirchange" then
+    core.on_dir_change(...)
   elseif type == "quit" then
     core.quit()
   end
@@ -1056,7 +1322,7 @@ function core.run()
   while true do
     core.frame_start = system.get_time()
     local did_redraw = core.step()
-    local need_more_work = run_threads()
+    local need_more_work = run_threads() or core.has_pending_rescan()
     if core.restart_request or core.quit_request then break end
     if not did_redraw and not need_more_work then
       idle_iterations = idle_iterations + 1

data/core/keymap.lua

@@ -210,6 +210,7 @@ keymap.add_direct {
   ["ctrl+a"] = "doc:select-all",
   ["ctrl+d"] = { "find-replace:select-add-next", "doc:select-word" },
   ["ctrl+f3"] = "find-replace:select-next",
+  ["ctrl+shift+f3"] = "find-replace:select-previous",
   ["ctrl+l"] = "doc:select-lines",
   ["ctrl+shift+l"] = { "find-replace:select-add-all", "doc:select-word" },
   ["ctrl+/"] = "doc:toggle-line-comments",

data/core/regex.lua

@@ -1,4 +1,3 @@
-
 -- So that in addition to regex.gsub(pattern, string), we can also do
 -- pattern:gsub(string).
 regex.__index = function(table, key) return regex[key]; end
@@ -6,7 +5,8 @@ regex.__index = function(table, key) return regex[key]; end
 regex.match = function(pattern_string, string, offset, options)
   local pattern = type(pattern_string) == "table" and
     pattern_string or regex.compile(pattern_string)
-  return regex.cmatch(pattern, string, offset or 1, options or 0)
+  local s, e = regex.cmatch(pattern, string, offset or 1, options or 0)
+  return s, e and e - 1
 end
 
 -- Will iterate back through any UTF-8 bytes so that we don't replace bits

data/core/rootview.lua

@@ -149,10 +149,17 @@ function Node:remove_view(root, view)
     else
       locked_size = locked_size_y
     end
-    if self.is_primary_node or locked_size then
+    local next_primary
+    if self.is_primary_node then
+      next_primary = core.root_view:select_next_primary_node()
+    end
+    if locked_size or (self.is_primary_node and not next_primary) then
       self.views = {}
       self:add_view(EmptyView())
     else
+      if other == next_primary then
+        next_primary = parent
+      end
       parent:consume(other)
       local p = parent
       while p.type ~= "leaf" do
@@ -160,7 +167,7 @@ function Node:remove_view(root, view)
       end
       p:set_active_view(p.active_view)
       if self.is_primary_node then
-        p.is_primary_node = true
+        next_primary.is_primary_node = true
       end
     end
   end
@@ -411,15 +418,8 @@ end
 -- calculating the sizes is the same for hsplits and vsplits, except the x/y
 -- axis are swapped; this function lets us use the same code for both
 local function calc_split_sizes(self, x, y, x1, x2, y1, y2)
-  local n
   local ds = ((x1 and x1 < 1) or (x2 and x2 < 1)) and 0 or style.divider_size
-  if x1 then
-    n = x1 + ds
-  elseif x2 then
-    n = self.size[x] - x2
-  else
-    n = math.floor(self.size[x] * self.divider)
-  end
+  local n = x1 and x1 + ds or (x2 and self.size[x] - x2 or math.floor(self.size[x] * self.divider))
   self.a.position[x] = self.position[x]
   self.a.position[y] = self.position[y]
   self.a.size[x] = n - ds
@@ -602,7 +602,7 @@ function Node:draw()
       self:draw_tabs()
     end
     local pos, size = self.active_view.position, self.active_view.size
-    core.push_clip_rect(pos.x, pos.y, size.x + pos.x % 1, size.y + pos.y % 1)
+    core.push_clip_rect(pos.x, pos.y, size.x, size.y)
     self.active_view:draw()
     core.pop_clip_rect()
   else
@@ -682,6 +682,10 @@ end
 
 
 function Node:resize(axis, value)
+  -- the application works fine with non-integer values but to have pixel-perfect
+  -- placements of view elements, like the scrollbar, we round the value to be
+  -- an integer.
+  value = math.floor(value)
   if self.type == 'leaf' then
     -- If it is not locked we don't accept the
     -- resize operation here because for proportional panes the resize is
@@ -826,6 +830,24 @@ function RootView:get_primary_node()
 end
 
 
+local function select_next_primary_node(node)
+  if node.is_primary_node then return end
+  if node.type ~= "leaf" then
+    return select_next_primary_node(node.a) or select_next_primary_node(node.b)
+  else
+    local lx, ly = node:get_locked_size()
+    if not lx and not ly then
+      return node
+    end
+  end
+end
+
+
+function RootView:select_next_primary_node()
+  return select_next_primary_node(self.root_node)
+end
+
+
 function RootView:open_doc(doc)
   local node = self:get_active_node_default()
   for i, view in ipairs(node.views) do

data/core/syntax.lua

@@ -22,7 +22,7 @@ end
 
 function syntax.get(filename, header)
   return find(filename, "files")
-      or find(header, "headers")
+      or (header and find(header, "headers"))
       or plain_text_syntax
 end
 

data/core/tokenizer.lua

@@ -1,4 +1,5 @@
 local syntax = require "core.syntax"
+local common = require "core.common"
 
 local tokenizer = {}
 
@@ -142,8 +143,13 @@ function tokenizer.tokenize(incoming_syntax, text, state)
       code = p._regex
     end    
     repeat
-      res = p.pattern and { text:find(at_start and "^" .. code or code, res[2]+1) } 
-        or { regex.match(code, text, res[2]+1, at_start and regex.ANCHORED or 0) }
+      local next = res[2] + 1
+      -- go to the start of the next utf-8 character
+      while text:byte(next) and common.is_utf8_cont(text, next) do
+        next = next + 1
+      end
+      res = p.pattern and { text:find(at_start and "^" .. code or code, next) }
+        or { regex.match(code, text, next, at_start and regex.ANCHORED or 0) }
       if res[1] and close and target[3] then
         local count = 0
         for i = res[1] - 1, 1, -1 do

data/core/view.lua

@@ -136,7 +136,7 @@ end
 function View:draw_background(color)
   local x, y = self.position.x, self.position.y
   local w, h = self.size.x, self.size.y
-  renderer.draw_rect(x, y, w + x % 1, h + y % 1, color)
+  renderer.draw_rect(x, y, w, h, color)
 end