Implement directory monitoring using septag/dmon

Use a notification based directory monitoring based on the
septag/dmon lirbary instead of periodically rescan the whole
project's tree.
This commit is contained in:
Francesco Abbate
2021-10-08 21:31:22 +02:00
parent 92362586df
commit 9c43727ebc
14 changed files with 2333 additions and 171 deletions
+347 -129
View File
@@ -52,13 +52,6 @@ local function update_recents_project(action, dir_path_abs)
end
function core.reschedule_project_scan()
if core.project_scan_thread_id then
core.threads[core.project_scan_thread_id].wake = 0
end
end
function core.set_project_dir(new_dir, change_project_fn)
local chdir_ok = pcall(system.chdir, new_dir)
if chdir_ok then
@@ -66,9 +59,6 @@ function core.set_project_dir(new_dir, change_project_fn)
core.project_dir = common.normalize_path(new_dir)
core.project_directories = {}
core.add_project_directory(new_dir)
core.project_files = {}
core.project_files_limit = false
core.reschedule_project_scan()
return true
end
return false
@@ -102,6 +92,20 @@ local function compare_file(a, b)
return a.filename < b.filename
end
-- compute a file's info entry completed with "filename" to be used
-- in project scan or falsy if it shouldn't appear in the list.
local function get_project_file_info(root, file)
local info = system.get_file_info(root .. file)
if info then
info.filename = strip_leading_path(file)
return (info.size < config.file_size_limit * 1e6 and
not common.match_pattern(info.filename, config.ignore_files)
and info)
end
end
-- "root" will by an absolute path without trailing '/'
-- "path" will be a path starting with '/' and without trailing '/'
-- or the empty string.
@@ -110,34 +114,27 @@ end
-- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'.
local function get_directory_files(root, path, t, recursive, begin_hook)
local function get_directory_files(dir, root, path, t, begin_hook, max_files)
if begin_hook then begin_hook() end
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
local dirs, files = {}, {}
local entries_count = 0
local max_entries = config.max_project_files
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
table.insert(info.type == "dir" and dirs or files, info)
entries_count = entries_count + 1
if recursive and entries_count > max_entries then return nil, entries_count end
end
local info = get_project_file_info(root, path .. PATHSEP .. file)
if info then
table.insert(info.type == "dir" and dirs or files, info)
entries_count = entries_count + 1
end
end
table.sort(dirs, compare_file)
for _, f in ipairs(dirs) do
table.insert(t, f)
if recursive and entries_count <= max_entries then
local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
entries_count = entries_count + subdir_count
f.scanned = true
if (not max_files or entries_count <= max_files) and core.project_subdir_is_shown(dir, f.filename) then
local sub_limit = max_files and max_files - entries_count
local _, n = get_directory_files(dir, root, PATHSEP .. f.filename, t, begin_hook, sub_limit)
entries_count = entries_count + n
end
end
@@ -149,132 +146,289 @@ local function get_directory_files(root, path, t, recursive, begin_hook)
return t, entries_count
end
local function project_scan_thread()
local function diff_files(a, b)
if #a ~= #b then return true end
for i, v in ipairs(a) do
if b[i].filename ~= v.filename
or b[i].modified ~= v.modified then
return true
end
end
end
while true do
-- get project files and replace previous table if the new table is
-- different
local i = 1
while not core.project_files_limit and i <= #core.project_directories do
local dir = core.project_directories[i]
local t, entries_count = get_directory_files(dir.name, "", {}, true)
if diff_files(dir.files, t) then
if entries_count > config.max_project_files then
core.project_files_limit = true
core.status_view:show_message("!", style.accent,
"Too many files in project directory: stopped reading at "..
config.max_project_files.." files. For more information see "..
"usage.md at github.com/franko/lite-xl."
)
end
dir.files = t
core.redraw = true
end
if dir.name == core.project_dir then
core.project_files = dir.files
end
i = i + 1
function core.project_subdir_set_show(dir, filename, show)
dir.shown_subdir[filename] = show
if dir.files_limit and PLATFORM == "Linux" then
local fullpath = dir.name .. PATHSEP .. filename
local watch_fn = show and system.watch_dir_add or system.watch_dir_rm
local success = watch_fn(dir.watch_id, fullpath)
if not success then
core.log("Internal warning: error calling system.watch_dir_%s", show and "add" or "rm")
end
-- wait for next scan
coroutine.yield(config.project_scan_rate)
end
end
function core.is_project_folder(dirname)
for _, dir in ipairs(core.project_directories) do
if dir.name == dirname then
return true
end
end
return false
function core.project_subdir_is_shown(dir, filename)
return not dir.files_limit or dir.shown_subdir[filename]
end
function core.scan_project_folder(dirname, filename)
for _, dir in ipairs(core.project_directories) do
if dir.name == dirname then
for i, file in ipairs(dir.files) do
local file = dir.files[i]
if file.filename == filename then
if file.scanned then return end
local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
for j, new_file in ipairs(new_files) do
table.insert(dir.files, i + j, new_file)
end
file.scanned = true
return
local function show_max_files_warning()
core.status_view:show_message("!", style.accent,
"Too many files in project directory: stopped reading at "..
config.max_project_files.." files. For more information see "..
"usage.md at github.com/franko/lite-xl."
)
end
-- Populate a project folder top directory by scanning the filesystem.
local function scan_project_folder(index)
local dir = core.project_directories[index]
local t, entries_count = get_directory_files(dir, dir.name, "", {}, nil, config.max_project_files)
if entries_count > config.max_project_files then
dir.files_limit = true
-- Watch non-recursively on Linux only.
-- The reason is recursively watching with dmon on linux
-- doesn't work on very large directories.
dir.watch_id = system.watch_dir(dir.name, PLATFORM ~= "Linux")
if core.status_view then -- May be not yet initialized.
show_max_files_warning()
end
else
dir.watch_id = system.watch_dir(dir.name, true)
end
dir.files = t
core.dir_rescan_add_job(dir, ".")
end
function core.add_project_directory(path)
-- top directories has a file-like "item" but the item.filename
-- will be simply the name of the directory, without its path.
-- The field item.topdir will identify it as a top level directory.
path = common.normalize_path(path)
local dir = {
name = path,
item = {filename = common.basename(path), type = "dir", topdir = true},
files_limit = false,
is_dirty = true,
shown_subdir = {},
}
table.insert(core.project_directories, dir)
scan_project_folder(#core.project_directories)
if path == core.project_dir then
core.project_files = dir.files
end
core.redraw = true
end
local function file_search(files, info)
local filename, type = info.filename, info.type
local inf, sup = 1, #files
while sup - inf > 8 do
local curr = math.floor((inf + sup) / 2)
if system.path_compare(filename, type, files[curr].filename, files[curr].type) then
sup = curr - 1
else
inf = curr
end
end
repeat
if files[inf].filename == filename then
return inf, true
end
inf = inf + 1
until inf > sup or system.path_compare(filename, type, files[inf].filename, files[inf].type)
return inf, false
end
local function project_scan_add_entry(dir, fileinfo)
local index, match = file_search(dir.files, fileinfo)
if not match then
table.insert(dir.files, index, fileinfo)
dir.is_dirty = true
end
end
local function files_info_equal(a, b)
return a.filename == b.filename and a.type == b.type
end
-- for "a" inclusive from i1 + 1 and i1 + n
local function files_list_match(a, i1, n, b)
if n ~= #b then return false end
for i = 1, n do
if not files_info_equal(a[i1 + i], b[i]) then
return false
end
end
return true
end
-- arguments like for files_list_match
local function files_list_replace(as, i1, n, bs)
local m = #bs
local i, j = 1, 1
while i <= m or i <= n do
local a, b = as[i1 + i], bs[j]
if i > n or (j <= m and not files_info_equal(a, b) and
not system.path_compare(a.filename, a.type, b.filename, b.type))
then
table.insert(as, i1 + i, b)
i, j, n = i + 1, j + 1, n + 1
elseif j > m or system.path_compare(a.filename, a.type, b.filename, b.type) then
table.remove(as, i1 + i)
n = n - 1
else
i, j = i + 1, j + 1
end
end
end
local function project_subdir_bounds(dir, filename)
local index, n = 0, #dir.files
for i, file in ipairs(dir.files) do
local file = dir.files[i]
if file.filename == filename then
index, n = i, #dir.files - i
for j = 1, #dir.files - i do
if not common.path_belongs_to(dir.files[i + j].filename, filename) then
n = j - 1
break
end
end
return index, n, file
end
end
end
local function rescan_project_subdir(dir, filename_rooted)
local new_files = get_directory_files(dir, dir.name, filename_rooted, {}, coroutine.yield)
local index, n = 0, #dir.files
if filename_rooted ~= "" then
local filename = strip_leading_path(filename_rooted)
index, n = project_subdir_bounds(dir, filename)
end
local function find_project_files_co(root, path)
local size_limit = config.file_size_limit * 10e5
if not files_list_match(dir.files, index, n, new_files) then
files_list_replace(dir.files, index, n, new_files)
dir.is_dirty = true
return true
end
end
function core.update_project_subdir(dir, filename, expanded)
local index, n, file = project_subdir_bounds(dir, filename)
if index then
local new_files = expanded and get_directory_files(dir, dir.name, PATHSEP .. filename, {}) or {}
files_list_replace(dir.files, index, n, new_files)
dir.is_dirty = true
return true
end
end
-- Find files and directories recursively reading from the filesystem.
-- Filter files and yields file's directory and info table. This latter
-- is filled to be like required by project directories "files" list.
local function find_files_rec(root, path)
local all = system.list_dir(root .. path) or {}
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
if info.type == "file" then
coroutine.yield(root, info)
else
find_project_files_co(root, PATHSEP .. info.filename)
end
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info then
info.filename = strip_leading_path(file)
if info.type == "file" then
coroutine.yield(root, info)
else
find_files_rec(root, PATHSEP .. info.filename)
end
end
end
end
-- Iterator function to list all project files
local function project_files_iter(state)
local dir = core.project_directories[state.dir_index]
state.file_index = state.file_index + 1
while dir and state.file_index > #dir.files do
state.dir_index = state.dir_index + 1
state.file_index = 1
dir = core.project_directories[state.dir_index]
if state.co then
-- We have a coroutine to fetch for files, use the coroutine.
-- Used for directories that exceeds the files nuumber limit.
local ok, name, file = coroutine.resume(state.co, dir.name, "")
if ok and name then
return name, file
else
-- The coroutine terminated, increment file/dir counter to scan
-- next project directory.
state.co = false
state.file_index = 1
state.dir_index = state.dir_index + 1
dir = core.project_directories[state.dir_index]
end
else
-- Increase file/dir counter
state.file_index = state.file_index + 1
while dir and state.file_index > #dir.files do
state.dir_index = state.dir_index + 1
state.file_index = 1
dir = core.project_directories[state.dir_index]
end
end
if not dir then return end
if dir.files_limit then
-- The current project directory is files limited: create a couroutine
-- to read files from the filesystem.
state.co = coroutine.create(find_files_rec)
return project_files_iter(state)
end
return dir.name, dir.files[state.file_index]
end
function core.get_project_files()
if core.project_files_limit then
return coroutine.wrap(function()
for _, dir in ipairs(core.project_directories) do
find_project_files_co(dir.name, "")
end
end)
else
local state = { dir_index = 1, file_index = 0 }
return project_files_iter, state
end
local state = { dir_index = 1, file_index = 0 }
return project_files_iter, state
end
function core.project_files_number()
if not core.project_files_limit then
local n = 0
for i = 1, #core.project_directories do
n = n + #core.project_directories[i].files
local n = 0
for i = 1, #core.project_directories do
if core.project_directories[i].files_limit then return end
n = n + #core.project_directories[i].files
end
return n
end
local function project_dir_by_watch_id(watch_id)
for i = 1, #core.project_directories do
if core.project_directories[i].watch_id == watch_id then
return core.project_directories[i]
end
return n
end
end
local function project_scan_remove_file(dir, filepath)
local fileinfo = { filename = filepath }
for _, filetype in ipairs {"dir", "file"} do
fileinfo.type = filetype
local index, match = file_search(dir.files, fileinfo)
if match then
table.remove(dir.files, index)
dir.is_dirty = true
return
end
end
end
local function project_scan_add_file(dir, filepath)
for fragment in string.gmatch(filepath, "([^/\\]+)") do
if common.match_pattern(fragment, config.ignore_files) then
return
end
end
local fileinfo = get_project_file_info(dir.name, PATHSEP .. filepath)
if fileinfo then
project_scan_add_entry(dir, fileinfo)
end
end
@@ -371,19 +525,6 @@ function core.load_user_directory()
end
function core.add_project_directory(path)
-- top directories has a file-like "item" but the item.filename
-- will be simply the name of the directory, without its path.
-- The field item.topdir will identify it as a top level directory.
path = common.normalize_path(path)
table.insert(core.project_directories, {
name = path,
item = {filename = common.basename(path), type = "dir", topdir = true},
files = {}
})
end
function core.remove_project_directory(path)
-- skip the fist directory because it is the project's directory
for i = 2, #core.project_directories do
@@ -519,7 +660,6 @@ function core.init()
cur_node = cur_node:split("down", core.command_view, {y = true})
cur_node = cur_node:split("down", core.status_view, {y = true})
core.project_scan_thread_id = core.add_thread(project_scan_thread)
command.add_defaults()
local got_user_error = not core.load_user_directory()
local plugins_success, plugins_refuse_list = core.load_plugins()
@@ -530,6 +670,12 @@ function core.init()
end
local got_project_error = not core.load_project_module()
-- We assume we have just a single project directory here. Now that StatusView
-- is there show max files warning if needed.
if core.project_directories[1].files_limit then
show_max_files_warning()
end
for _, filename in ipairs(files) do
core.root_view:open_doc(core.open_doc(filename))
end
@@ -918,6 +1064,76 @@ function core.try(fn, ...)
return false, err
end
local scheduled_rescan = {}
function core.has_pending_rescan()
for _ in pairs(scheduled_rescan) do
return true
end
end
function core.dir_rescan_add_job(dir, filepath)
local dirpath = filepath:match("^(.+)[/\\].+$")
local dirpath_rooted = dirpath and PATHSEP .. dirpath or ""
local abs_dirpath = dir.name .. dirpath_rooted
if dirpath then
-- check if the directory is in the project files list, if not exit
local dir_index, dir_match = file_search(dir.files, {filename = dirpath, type = "dir"})
-- Note that is dir_match is false dir_index greaten than the last valid index.
-- We use dir_index to index dir.files below only if dir_match is true.
if not dir_match or not core.project_subdir_is_shown(dir, dir.files[dir_index].filename) then return end
end
local new_time = system.get_time() + 1
-- evaluate new rescan request versus existing rescan
local remove_list = {}
for _, rescan in pairs(scheduled_rescan) do
if abs_dirpath == rescan.abs_path or common.path_belongs_to(abs_dirpath, rescan.abs_path) then
-- abs_dirpath is a subpath of a scan already ongoing: skip
rescan.time_limit = new_time
return
elseif common.path_belongs_to(rescan.abs_path, abs_dirpath) then
-- abs_dirpath already cover this rescan: add to the list of rescan to be removed
table.insert(remove_list, rescan.abs_path)
end
end
for _, key_path in ipairs(remove_list) do
scheduled_rescan[key_path] = nil
end
scheduled_rescan[abs_dirpath] = {dir = dir, path = dirpath_rooted, abs_path = abs_dirpath, time_limit = new_time}
core.add_thread(function()
while true do
local rescan = scheduled_rescan[abs_dirpath]
if not rescan then return end
if system.get_time() > rescan.time_limit then
local has_changes = rescan_project_subdir(rescan.dir, rescan.path)
if has_changes then
core.redraw = true -- we run without an event, from a thread
rescan.time_limit = new_time
else
scheduled_rescan[rescan.abs_path] = nil
return
end
end
coroutine.yield(0.2)
end
end)
end
function core.on_dir_change(watch_id, action, filepath)
local dir = project_dir_by_watch_id(watch_id)
if not dir then return end
core.dir_rescan_add_job(dir, filepath)
if action == "delete" then
project_scan_remove_file(dir, filepath)
elseif action == "create" then
project_scan_add_file(dir, filepath)
end
end
function core.on_event(type, ...)
local did_keymap = false
@@ -954,6 +1170,8 @@ function core.on_event(type, ...)
end
elseif type == "focuslost" then
core.root_view:on_focus_lost(...)
elseif type == "dirchange" then
core.on_dir_change(...)
elseif type == "quit" then
core.quit()
end
@@ -1060,7 +1278,7 @@ function core.run()
while true do
core.frame_start = system.get_time()
local did_redraw = core.step()
local need_more_work = run_threads()
local need_more_work = run_threads() or core.has_pending_rescan()
if core.restart_request or core.quit_request then break end
if not did_redraw and not need_more_work then
idle_iterations = idle_iterations + 1