jgmdev
b0c005a5ac
* Introduces a flag that syntax writers can turn off named space_handling, turning it off means that your syntax will take care of handling the excessive amount of spaces that can slow down the tokenizer. * Adds another pattern at the end of every single table that also improves tokenizer performance by matching words that weren't match by any of the synxtax patterns. * Modifies language_md to turn off the provided space_handling and do its own since it has rules that require a space at the beginning, also handles long consecutives amount of dashes used in tables that degrade performance. * This changes where discussed in collaboration with @Guldoman and @takase1121 thanks to all!
49 lines
1.5 KiB
Lua
49 lines
1.5 KiB
Lua
local common = require "core.common"
|
|
|
|
local syntax = {}
|
|
syntax.items = {}
|
|
|
|
local plain_text_syntax = { name = "Plain Text", patterns = {}, symbols = {} }
|
|
|
|
|
|
function syntax.add(t)
|
|
if type(t.space_handling) ~= "boolean" then t.space_handling = true end
|
|
|
|
if t.patterns then
|
|
-- the rule %s+ gives us a performance gain for the tokenizer in lines with
|
|
-- long amounts of consecutive spaces, can be disabled by plugins where it
|
|
-- causes conflicts by declaring the table property: space_handling = false
|
|
if t.space_handling then
|
|
table.insert(t.patterns, { pattern = "%s+", type = "normal" })
|
|
end
|
|
|
|
-- this rule gives us additional performance gain by matching every word
|
|
-- that was not matched by the syntax patterns as a single token, preventing
|
|
-- the tokenizer from iterating over each character individually which is a
|
|
-- lot slower since iteration occurs in lua instead of C and adding to that
|
|
-- it will also try to match every pattern to a single char (same as spaces)
|
|
table.insert(t.patterns, { pattern = "%w+%f[%s]", type = "normal" })
|
|
end
|
|
|
|
table.insert(syntax.items, t)
|
|
end
|
|
|
|
|
|
local function find(string, field)
|
|
for i = #syntax.items, 1, -1 do
|
|
local t = syntax.items[i]
|
|
if common.match_pattern(string, t[field] or {}) then
|
|
return t
|
|
end
|
|
end
|
|
end
|
|
|
|
function syntax.get(filename, header)
|
|
return find(filename, "files")
|
|
or (header and find(header, "headers"))
|
|
or plain_text_syntax
|
|
end
|
|
|
|
|
|
return syntax
|