vhaudiquet/deb-lite-xl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement lazy loading of directories

Browse Source 
When the number of files in a project directory is above the max
limit switch back to a mechanism to read directory content only
when the corresponding folder is expanded in the treeview.

When the command core:find-file is invoked the command core:open-file
is executed instead because the complete list of the project's
files is not available.

When a project search is done we search through all the files within
the project dir without indexing them.

Address issues #217 #203 #183.
This commit is contained in:
Francesco Abbate
2021-05-26 14:22:10 +02:00
parent 8acb3fae8c
commit 10fde6e264
4 changed files with 146 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/core/commands/core.lua
+3
View File
@@ -66,6 +66,9 @@ command.add(nil, {
end,
["core:find-file"] = function()
if core.project_files_limit then
return command.perform "core:open-file"
end
local files = {}
for dir, item in core.get_project_files() do
if item.type == "file" then
data/core/init.lua
+113 -59
View File
@@ -72,6 +72,7 @@ function core.set_project_dir(new_dir, change_project_fn)
core.project_directories = {}
core.add_project_directory(new_dir)
core.project_files = {}
core.project_files_limit = false
core.reschedule_project_scan()
return true
end
@@ -99,6 +100,57 @@ local function strip_trailing_slash(filename)
return filename
end
local function compare_file(a, b)
return a.filename < b.filename
end
-- "root" will by an absolute path without trailing '/'
-- "path" will be a path starting with '/' and without trailing '/'
-- or the empty string.
-- It will identifies a sub-path within "root.
-- The current path location will therefore always be: root .. path.
-- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'.
local function get_directory_files(root, path, t, recursive, begin_hook)
if begin_hook then begin_hook() end
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
local dirs, files = {}, {}
local entries_count = 0
local max_entries = config.max_project_files
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
table.insert(info.type == "dir" and dirs or files, info)
entries_count = entries_count + 1
if recursive and entries_count > max_entries then return nil, entries_count end
end
end
end
table.sort(dirs, compare_file)
for _, f in ipairs(dirs) do
table.insert(t, f)
if recursive and entries_count <= max_entries then
local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
entries_count = entries_count + subdir_count
f.scanned = true
end
end
table.sort(files, compare_file)
for _, f in ipairs(files) do
table.insert(t, f)
end
return t, entries_count
end
local function project_scan_thread()
local function diff_files(a, b)
if #a ~= #b then return true end
@@ -110,65 +162,16 @@ local function project_scan_thread()
end
end
local function compare_file(a, b)
return a.filename < b.filename
end
-- "root" will by an absolute path without trailing '/'
-- "path" will be a path starting with '/' and without trailing '/'
-- or the empty string.
-- It will identifies a sub-path within "root.
-- The current path location will therefore always be: root .. path.
-- When recursing "root" will always be the same, only "path" will change.
-- Returns a list of file "items". In eash item the "filename" will be the
-- complete file path relative to "root" *without* the trailing '/'.
local function get_files(root, path, t)
coroutine.yield()
t = t or {}
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
local dirs, files = {}, {}
local entries_count = 0
local max_entries = config.max_project_files
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
table.insert(info.type == "dir" and dirs or files, info)
entries_count = entries_count + 1
if entries_count > max_entries then break end
end
end
end
table.sort(dirs, compare_file)
for _, f in ipairs(dirs) do
table.insert(t, f)
if entries_count <= max_entries then
local subdir_t, subdir_count = get_files(root, PATHSEP .. f.filename, t)
entries_count = entries_count + subdir_count
end
end
table.sort(files, compare_file)
for _, f in ipairs(files) do
table.insert(t, f)
end
return t, entries_count
end
while true do
-- get project files and replace previous table if the new table is
-- different
for i = 1, #core.project_directories do
local i = 1
while not core.project_files_limit and i <= #core.project_directories do
local dir = core.project_directories[i]
local t, entries_count = get_files(dir.name, "")
local t, entries_count = get_directory_files(dir.name, "", {}, true)
if diff_files(dir.files, t) then
if entries_count > config.max_project_files then
core.project_files_limit = true
core.status_view:show_message("!", style.accent,
"Too many files in project directory: stopping reading at "..
config.max_project_files.." files according to config.max_project_files. "..
@@ -182,6 +185,7 @@ local function project_scan_thread()
if dir.name == core.project_dir then
core.project_files = dir.files
end
i = i + 1
end
-- wait for next scan
@@ -190,6 +194,46 @@ local function project_scan_thread()
end
function core.scan_project_folder(dirname, filename)
for _, dir in ipairs(core.project_directories) do
if dir.name == dirname then
for i, file in ipairs(dir.files) do
local file = dir.files[i]
if file.filename == filename then
if file.scanned then return end
local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
for j, new_file in ipairs(new_files) do
table.insert(dir.files, i + j, new_file)
end
file.scanned = true
return
end
end
end
end
end
local function find_project_files_co(root, path)
local size_limit = config.file_size_limit * 10e5
local all = system.list_dir(root .. path) or {}
for _, file in ipairs(all) do
if not common.match_pattern(file, config.ignore_files) then
local file = path .. PATHSEP .. file
local info = system.get_file_info(root .. file)
if info and info.size < size_limit then
info.filename = strip_leading_path(file)
if info.type == "file" then
coroutine.yield(root, info)
else
find_project_files_co(root, PATHSEP .. info.filename)
end
end
end
end
end
local function project_files_iter(state)
local dir = core.project_directories[state.dir_index]
state.file_index = state.file_index + 1
@@ -204,17 +248,27 @@ end
function core.get_project_files()
local state = { dir_index = 1, file_index = 0 }
return project_files_iter, state
if core.project_files_limit then
return coroutine.wrap(function()
for _, dir in ipairs(core.project_directories) do
find_project_files_co(dir.name, "")
end
end)
else
local state = { dir_index = 1, file_index = 0 }
return project_files_iter, state
end
end
function core.project_files_number()
local n = 0
for i = 1, #core.project_directories do
n = n + #core.project_directories[i].files
if not core.project_files_limit then
local n = 0
for i = 1, #core.project_directories do
n = n + #core.project_directories[i].files
end
return n
end
return n
end