From ac4995ee46ef7059479f9c5b90ecbd65ba092d85 Mon Sep 17 00:00:00 2001 From: Guldoman Date: Tue, 22 Oct 2024 18:33:01 +0200 Subject: [PATCH] Fix multi-type usage in delimited patterns (#1740) In the "end" pattern we weren't considering the multiple types. --- data/core/tokenizer.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/data/core/tokenizer.lua b/data/core/tokenizer.lua index 0e769b2..263da5b 100644 --- a/data/core/tokenizer.lua +++ b/data/core/tokenizer.lua @@ -283,7 +283,8 @@ function tokenizer.tokenize(incoming_syntax, text, state, resume) -- continue trying to match the end pattern of a pair if we have a state set if current_pattern_idx > 0 then local p = current_syntax.patterns[current_pattern_idx] - local s, e = find_text(text, p, i, false, true) + local find_results = { find_text(text, p, i, false, true) } + local s, e = find_results[1], find_results[2] local cont = true -- If we're in subsyntax mode, always check to see if we end our syntax @@ -305,7 +306,12 @@ function tokenizer.tokenize(incoming_syntax, text, state, resume) -- continue on as normal. if cont then if s then - push_token(res, p.type, text:usub(i, e)) + -- Push remaining token before the end delimiter + if s > i then + push_token(res, p.type, text:usub(i, s - 1)) + end + -- Push the end delimiter + push_tokens(res, current_syntax, p, text, find_results) set_subsyntax_pattern_idx(0) i = e + 1 else