Fix problem checking utf-8 cont at end of string
This commit is contained in:
@@ -145,7 +145,7 @@ function tokenizer.tokenize(incoming_syntax, text, state)
|
|||||||
repeat
|
repeat
|
||||||
local next = res[2] + 1
|
local next = res[2] + 1
|
||||||
-- go to the start of the next utf-8 character
|
-- go to the start of the next utf-8 character
|
||||||
while common.is_utf8_cont(text, next) do
|
while text:byte(next) and common.is_utf8_cont(text, next) do
|
||||||
next = next + 1
|
next = next + 1
|
||||||
end
|
end
|
||||||
res = p.pattern and { text:find(at_start and "^" .. code or code, next) }
|
res = p.pattern and { text:find(at_start and "^" .. code or code, next) }
|
||||||
|
|||||||
Reference in New Issue
Block a user