From 5cdd8009109409cc45daff7b5364e5d43da95ca0 Mon Sep 17 00:00:00 2001 From: Francesco Abbate Date: Sat, 23 Oct 2021 15:03:09 +0200 Subject: [PATCH] Fix problem checking utf-8 cont at end of string --- data/core/tokenizer.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/core/tokenizer.lua b/data/core/tokenizer.lua index 08a5ea3..d95baeb 100644 --- a/data/core/tokenizer.lua +++ b/data/core/tokenizer.lua @@ -145,7 +145,7 @@ function tokenizer.tokenize(incoming_syntax, text, state) repeat local next = res[2] + 1 -- go to the start of the next utf-8 character - while common.is_utf8_cont(text, next) do + while text:byte(next) and common.is_utf8_cont(text, next) do next = next + 1 end res = p.pattern and { text:find(at_start and "^" .. code or code, next) }