Add utf8 support to tokenizer (#945)

* add utf8 support to tokenizer

* wrap utf8 functions in string table using a 'u' prefix

* document new utf8 functions

src/api/api.c

@@ -5,6 +5,7 @@ int luaopen_renderer(lua_State *L);
 int luaopen_regex(lua_State *L);
 int luaopen_process(lua_State *L);
 int luaopen_dirmonitor(lua_State* L);
+int luaopen_utf8(lua_State* L);
 
 static const luaL_Reg libs[] = {
   { "system",     luaopen_system     },
@@ -12,6 +13,7 @@ static const luaL_Reg libs[] = {
   { "regex",      luaopen_regex      },
   { "process",    luaopen_process    },
   { "dirmonitor", luaopen_dirmonitor },
+  { "utf8",       luaopen_utf8       },
   { NULL, NULL }
 };
 

src/meson.build

@@ -4,6 +4,7 @@ lite_sources = [
     'api/regex.c',
     'api/system.c',
     'api/process.c',
+    'api/utf8.c',
     'renderer.c',
     'renwindow.c',
     'rencache.c',