add utf8 module

14 files changed, 690 insertions, 0 deletions

diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index d81828f1f..90e583c93 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -279,6 +279,13 @@ tb_int_t xm_winos_registry_values(lua_State *lua);
 tb_int_t xm_winos_short_path(lua_State *lua);
 #endif
 
+// the utf8 functions
+tb_int_t xm_utf8_len(lua_State *lua);
+tb_int_t xm_utf8_char(lua_State *lua);
+tb_int_t xm_utf8_codepoint(lua_State *lua);
+tb_int_t xm_utf8_offset(lua_State *lua);
+tb_int_t xm_utf8_codes(lua_State *lua);
+
 // the string functions
 tb_int_t xm_string_trim(lua_State *lua);
 tb_int_t xm_string_split(lua_State *lua);
@@ -583,6 +590,16 @@ static luaL_Reg const g_bloom_filter_functions[] = {
     { tb_null, tb_null },
 };
 
+// the utf8 functions
+static luaL_Reg const g_utf8_functions[] = {
+    {"char", xm_utf8_char},
+    {"codes", xm_utf8_codes},
+    {"codepoint", xm_utf8_codepoint},
+    {"len", xm_utf8_len},
+    {"offset", xm_utf8_offset},
+    {tb_null, tb_null}
+};
+
 // the string functions
 static luaL_Reg const g_string_functions[] = {
     { "trim", xm_string_trim },
@@ -722,6 +739,9 @@ static luaL_Reg const g_thread_functions[] = {
     { tb_null, tb_null },
 };
 
+// the utf8 functions
+
+
 // the lua global instance for signal handler
 static lua_State *g_lua = tb_null;
 
@@ -1547,6 +1567,12 @@ xm_engine_ref_t xm_engine_init(tb_char_t const *name, xm_engine_lni_initalizer_c
         // bind base64 functions
         xm_lua_register(engine->lua, "base64", g_base64_functions);
 
+        // bind utf8 functions
+        xm_lua_register(engine->lua, "utf8", g_utf8_functions);
+        lua_pushstring(engine->lua, "[\0-\x7F\xC2-\xFD][\x80-\xBF]*");
+        lua_setfield(engine->lua, -2, "charpattern");
+        lua_pop(engine->lua, 1);
+
         // bind string functions
         xm_lua_register(engine->lua, "string", g_string_functions);
 
diff --git a/core/src/xmake/utf8/char.c b/core/src/xmake/utf8/char.c
new file mode 100644
index 000000000..afb0a0e0f
--- /dev/null
+++ b/core/src/xmake/utf8/char.c
@@ -0,0 +1,65 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        char.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * private implementation
+ */
+
+static void xm_utf8_char_push(lua_State *lua, tb_int_t arg) {
+    lua_Unsigned code = (lua_Unsigned)luaL_checkinteger(lua, arg);
+    luaL_argcheck(lua, code <= XM_UTF8_MAXUTF, arg, "value out of range");
+    
+    tb_char_t buf[8];
+    tb_size_t n = xm_utf8_encode(buf, (xm_utf8_int_t)code);
+    if (n > 0)
+        lua_pushlstring(lua, buf, n);
+    else
+        luaL_error(lua, "value out of range");
+}
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+/*
+** utfchar(n1, n2, ...)  -> char(n1)..char(n2)...
+*/
+tb_int_t xm_utf8_char(lua_State *lua) {
+    tb_int_t n = lua_gettop(lua);  /* number of arguments */
+    if (n == 1)  /* optimize common case of single char */
+        xm_utf8_char_push(lua, 1);
+    else {
+        tb_int_t i;
+        luaL_Buffer b;
+        luaL_buffinit(lua, &b);
+        for (i = 1; i <= n; i++) {
+            xm_utf8_char_push(lua, i);
+            luaL_addvalue(&b);
+        }
+        luaL_pushresult(&b);
+    }
+    return 1;
+}
diff --git a/core/src/xmake/utf8/codepoint.c b/core/src/xmake/utf8/codepoint.c
new file mode 100644
index 000000000..f4c9544f2
--- /dev/null
+++ b/core/src/xmake/utf8/codepoint.c
@@ -0,0 +1,62 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        codepoint.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+static tb_bool_t xm_utf8_codepoint_cb(xm_utf8_int_t code, tb_cpointer_t udata) {
+    lua_State* lua = (lua_State*)udata;
+    lua_pushinteger(lua, code);
+    return tb_true;
+}
+
+/*
+** codepoint(s, [i, [j [, lax]]]) -> returns codepoints for all
+** characters that start in the range [i,j]
+*/
+tb_int_t xm_utf8_codepoint(lua_State *lua) {
+    size_t len;
+    tb_char_t const* s = luaL_checklstring(lua, 1, &len);
+    lua_Integer posi = xm_utf8_posrelat(luaL_optinteger(lua, 2, 1), len);
+    lua_Integer pose = xm_utf8_posrelat(luaL_optinteger(lua, 3, posi), len);
+    tb_bool_t lax = lua_toboolean(lua, 4);
+
+    luaL_argcheck(lua, posi >= 1, 2, "out of bounds");
+    luaL_argcheck(lua, pose <= (lua_Integer)len, 3, "out of bounds");
+
+    if (posi > pose) return 0;  /* empty interval; return no values */
+    if (pose - posi >= INT_MAX)  /* (lua_Integer -> int) overflow? */
+        return luaL_error(lua, "string slice too long");
+    
+    tb_int_t n = (tb_int_t)(pose - posi) + 1;
+    luaL_checkstack(lua, n, "string slice too long");
+
+    if (!xm_utf8_codepoint_impl(s, len, posi, pose, !lax, xm_utf8_codepoint_cb, lua))
+        return luaL_error(lua, XM_UTF8_MSGInvalid);
+    
+    return n;
+}
diff --git a/core/src/xmake/utf8/codes.c b/core/src/xmake/utf8/codes.c
new file mode 100644
index 000000000..be6cf462a
--- /dev/null
+++ b/core/src/xmake/utf8/codes.c
@@ -0,0 +1,71 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        codes.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * private implementation
+ */
+
+static tb_int_t iter_aux (lua_State *lua, tb_bool_t strict) {
+    size_t len;
+    tb_char_t const* s = luaL_checklstring(lua, 1, &len);
+    lua_Unsigned n = (lua_Unsigned)lua_tointeger(lua, 2);
+    if (n < len) {
+        while (n < len && xm_utf8_iscontp(s + n)) n++;  /* go to next character */
+    }
+    if (n >= len)  /* (also handles original 'n' being negative) */
+        return 0;  /* no more codepoints */
+    else {
+        xm_utf8_int_t code;
+        tb_char_t const* next = xm_utf8_decode(s + n, &code, strict);
+        if (next == NULL || xm_utf8_iscontp(next))
+            return luaL_error(lua, XM_UTF8_MSGInvalid);
+        lua_pushinteger(lua, n + 1);
+        lua_pushinteger(lua, code);
+        return 2;
+    }
+}
+
+static int iter_auxstrict (lua_State *lua) {
+    return iter_aux(lua, tb_true);
+}
+
+static int iter_auxlax (lua_State *lua) {
+    return iter_aux(lua, tb_false);
+}
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+tb_int_t xm_utf8_codes(lua_State *lua) {
+    tb_bool_t lax = lua_toboolean(lua, 2);
+    tb_char_t const* s = luaL_checkstring(lua, 1);
+    luaL_argcheck(lua, !xm_utf8_iscontp(s), 1, XM_UTF8_MSGInvalid);
+    lua_pushcfunction(lua, lax ? iter_auxlax : iter_auxstrict);
+    lua_pushvalue(lua, 1);
+    lua_pushinteger(lua, 0);
+    return 3;
+}
diff --git a/core/src/xmake/utf8/len.c b/core/src/xmake/utf8/len.c
new file mode 100644
index 000000000..410c0a1ef
--- /dev/null
+++ b/core/src/xmake/utf8/len.c
@@ -0,0 +1,54 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        len.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+/*
+** utf8len(s [, i [, j [, lax]]]) --> number of characters that
+** start in the range [i,j], or nil + current position if 's' is not
+** well formed in that interval
+*/
+tb_int_t xm_utf8_len(lua_State *lua) {
+    size_t len;
+    tb_char_t const* s = luaL_checklstring(lua, 1, &len);
+    lua_Integer posi = xm_utf8_posrelat(luaL_optinteger(lua, 2, 1), len);
+    lua_Integer posj = xm_utf8_posrelat(luaL_optinteger(lua, 3, -1), len);
+    tb_bool_t lax = lua_toboolean(lua, 4);
+    luaL_argcheck(lua, 1 <= posi && --posi <= (lua_Integer)len, 2, "initial position out of bounds");
+    luaL_argcheck(lua, --posj < (lua_Integer)len, 3, "final position out of bounds");
+
+    tb_size_t errpos = 0;
+    tb_long_t n = xm_utf8_len_impl(s, len, posi + 1, posj + 1, !lax, &errpos);
+    if (n < 0) {
+        lua_pushnil(lua);
+        lua_pushinteger(lua, errpos);
+        return 2;
+    }
+    lua_pushinteger(lua, n);
+    return 1;
+}
diff --git a/core/src/xmake/utf8/offset.c b/core/src/xmake/utf8/offset.c
new file mode 100644
index 000000000..89635eac1
--- /dev/null
+++ b/core/src/xmake/utf8/offset.c
@@ -0,0 +1,53 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        offset.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+/*
+** offset(s, n, [i])  -> index where n-th character counting from
+**   position 'i' starts; 0 means character at 'i'.
+*/
+tb_int_t xm_utf8_offset(lua_State *lua) {
+    size_t len;
+    tb_char_t const* s = luaL_checklstring(lua, 1, &len);
+    lua_Integer n  = luaL_checkinteger(lua, 2);
+    lua_Integer posi = (n >= 0) ? 1 : len + 1;
+    posi = xm_utf8_posrelat(luaL_optinteger(lua, 3, posi), len);
+
+    tb_long_t result = xm_utf8_offset_impl(s, len, n, posi);
+    if (result == -1)
+        return luaL_argerror(lua, 3, "position out of bounds");
+    if (result == -2)
+        return luaL_error(lua, "initial position is a continuation byte");
+    if (result == 0) {
+        lua_pushnil(lua);
+        return 1;
+    }
+    lua_pushinteger(lua, result);
+    return 1;
+}
diff --git a/core/src/xmake/utf8/prefix.h b/core/src/xmake/utf8/prefix.h
new file mode 100644
index 000000000..7a0881248
--- /dev/null
+++ b/core/src/xmake/utf8/prefix.h
@@ -0,0 +1,9 @@
+#ifndef XM_UTF8_PREFIX_H
+#define XM_UTF8_PREFIX_H
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "../prefix.h"
+
+#endif
diff --git a/core/src/xmake/utf8/utf8.c b/core/src/xmake/utf8/utf8.c
new file mode 100644
index 000000000..8e164e0a5
--- /dev/null
+++ b/core/src/xmake/utf8/utf8.c
@@ -0,0 +1,160 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        utf8.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+tb_char_t const* xm_utf8_decode(tb_char_t const* s, xm_utf8_int_t* val, tb_bool_t strict) {
+    static const xm_utf8_int_t limits[] = {~(xm_utf8_int_t)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u};
+    tb_uint32_t c = (tb_byte_t)s[0];
+    xm_utf8_int_t res = 0; 
+    if (c < 0x80) 
+        res = c;
+    else {
+        tb_int_t count = 0; 
+        for (; c & 0x40; c <<= 1) { 
+            tb_uint32_t cc = (tb_byte_t)s[++count]; 
+            if (!xm_utf8_iscont(cc)) 
+                return tb_null; 
+            res = (res << 6) | (cc & 0x3F); 
+        }
+        res |= ((xm_utf8_int_t)(c & 0x7F) << (count * 5)); 
+        if (count > 5 || res > XM_UTF8_MAXUTF || res < limits[count])
+            return tb_null; 
+        s += count; 
+    }
+    if (strict) {
+        if (res > XM_UTF8_MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu))
+            return tb_null;
+    }
+    if (val) *val = res;
+    return s + 1; 
+}
+
+tb_size_t xm_utf8_encode(tb_char_t* s, xm_utf8_int_t val) {
+    if (val < 0x80) {
+        s[0] = (tb_char_t)val;
+        return 1;
+    }
+    if (val < 0x800) {
+        s[0] = (tb_char_t)(0xc0 | ((val >> 6) & 0x1f));
+        s[1] = (tb_char_t)(0x80 | (val & 0x3f));
+        return 2;
+    }
+    if (val < 0x10000) {
+        s[0] = (tb_char_t)(0xe0 | ((val >> 12) & 0x0f));
+        s[1] = (tb_char_t)(0x80 | ((val >> 6) & 0x3f));
+        s[2] = (tb_char_t)(0x80 | (val & 0x3f));
+        return 3;
+    }
+    if (val <= 0x10FFFF) {
+        s[0] = (tb_char_t)(0xf0 | ((val >> 18) & 0x07));
+        s[1] = (tb_char_t)(0x80 | ((val >> 12) & 0x3f));
+        s[2] = (tb_char_t)(0x80 | ((val >> 6) & 0x3f));
+        s[3] = (tb_char_t)(0x80 | (val & 0x3f));
+        return 4;
+    }
+    if (val <= 0x3FFFFFF) {
+        s[0] = (tb_char_t)(0xf8 | ((val >> 24) & 0x03));
+        s[1] = (tb_char_t)(0x80 | ((val >> 18) & 0x3f));
+        s[2] = (tb_char_t)(0x80 | ((val >> 12) & 0x3f));
+        s[3] = (tb_char_t)(0x80 | ((val >> 6) & 0x3f));
+        s[4] = (tb_char_t)(0x80 | (val & 0x3f));
+        return 5;
+    }
+    if (val <= 0x7FFFFFFF) {
+        s[0] = (tb_char_t)(0xfc | ((val >> 30) & 0x01));
+        s[1] = (tb_char_t)(0x80 | ((val >> 24) & 0x3f));
+        s[2] = (tb_char_t)(0x80 | ((val >> 18) & 0x3f));
+        s[3] = (tb_char_t)(0x80 | ((val >> 12) & 0x3f));
+        s[4] = (tb_char_t)(0x80 | ((val >> 6) & 0x3f));
+        s[5] = (tb_char_t)(0x80 | (val & 0x3f));
+        return 6;
+    }
+    return 0;
+}
+
+tb_long_t xm_utf8_len_impl(tb_char_t const* s, tb_size_t len, tb_long_t posi, tb_long_t posj, tb_bool_t strict, tb_size_t* errpos) {
+    tb_long_t n = 0;
+    while (posi <= posj) {
+        tb_char_t const* s1 = xm_utf8_decode(s + posi - 1, tb_null, strict);
+        if (s1 == tb_null) {
+            if (errpos) *errpos = posi;
+            return -1;
+        }
+        posi = s1 - s + 1;
+        n++;
+    }
+    return n;
+}
+
+tb_long_t xm_utf8_offset_impl(tb_char_t const* s, tb_size_t len, tb_long_t n, tb_long_t posi) {
+    // check
+    if (1 > posi || --posi > (tb_long_t)len) 
+        return -1; // error: position out of bounds
+
+    if (n == 0) {
+        // find beginning of current byte sequence
+        while (posi > 0 && xm_utf8_iscontp(s + posi)) posi--;
+    } else {
+        if (xm_utf8_iscontp(s + posi))
+            return -2; // error: initial position is a continuation byte
+        
+        if (n < 0) {
+            while (n < 0 && posi > 0) {
+                do {
+                    posi--;
+                } while (posi > 0 && xm_utf8_iscontp(s + posi));
+                n++;
+            }
+        } else {
+            n--;
+            while (n > 0 && posi < (tb_long_t)len) {
+                do {
+                    posi++;
+                } while (xm_utf8_iscontp(s + posi));
+                n--;
+            }
+        }
+    }
+
+    if (n == 0) return posi + 1;
+    return 0; // nil
+}
+
+tb_bool_t xm_utf8_codepoint_impl(tb_char_t const* s, tb_size_t len, tb_long_t posi, tb_long_t posj, tb_bool_t strict, xm_utf8_codepoint_func_t func, tb_cpointer_t udata) {
+    if (posi > posj) return tb_true; 
+    
+    tb_char_t const* se = s + posj;
+    for (s += posi - 1; s < se;) {
+        xm_utf8_int_t code;
+        s = xm_utf8_decode(s, &code, strict);
+        if (s == tb_null) return tb_false;
+        if (func && !func(code, udata)) return tb_false;
+    }
+    return tb_true;
+}
diff --git a/core/src/xmake/utf8/utf8.h b/core/src/xmake/utf8/utf8.h
new file mode 100644
index 000000000..8306abee9
--- /dev/null
+++ b/core/src/xmake/utf8/utf8.h
@@ -0,0 +1,48 @@
+#ifndef XM_UTF8_UTF8_H
+#define XM_UTF8_UTF8_H
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "prefix.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * macros
+ */
+#define xm_utf8_iscont(c)    (((c) & 0xC0) == 0x80)
+#define xm_utf8_iscontp(p)   xm_utf8_iscont(*(p))
+
+#define XM_UTF8_MAXUNICODE  0x10FFFFu
+#define XM_UTF8_MAXUTF      0x7FFFFFFFu
+#define XM_UTF8_MSGInvalid  "invalid UTF-8 code"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * types
+ */
+typedef tb_uint32_t xm_utf8_int_t;
+typedef tb_bool_t (*xm_utf8_codepoint_func_t)(xm_utf8_int_t code, tb_cpointer_t udata);
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * interfaces
+ */
+
+static __tb_inline__ tb_long_t xm_utf8_posrelat(tb_long_t pos, tb_size_t len) {
+    if (pos >= 0) return pos;
+    else if (0u - (tb_size_t)pos > len) return 0;
+    else return (tb_long_t)len + pos + 1;
+}
+
+tb_char_t const*    xm_utf8_decode(tb_char_t const* s, xm_utf8_int_t* val, tb_bool_t strict);
+tb_size_t           xm_utf8_encode(tb_char_t* s, xm_utf8_int_t val);
+
+tb_long_t           xm_utf8_len_impl(tb_char_t const* s, tb_size_t len, tb_long_t posi, tb_long_t posj, tb_bool_t strict, tb_size_t* errpos);
+tb_long_t           xm_utf8_offset_impl(tb_char_t const* s, tb_size_t len, tb_long_t n, tb_long_t posi);
+tb_bool_t           xm_utf8_codepoint_impl(tb_char_t const* s, tb_size_t len, tb_long_t posi, tb_long_t posj, tb_bool_t strict, xm_utf8_codepoint_func_t func, tb_cpointer_t udata);
+
+tb_int_t            xm_utf8_char(lua_State* lua);
+tb_int_t            xm_utf8_codes(lua_State* lua);
+tb_int_t            xm_utf8_codepoint(lua_State* lua);
+tb_int_t            xm_utf8_len(lua_State* lua);
+tb_int_t            xm_utf8_offset(lua_State* lua);
+
+#endif
diff --git a/core/src/xmake/xmake.sh b/core/src/xmake/xmake.sh
index 291001306..dfe533aa3 100755
--- a/core/src/xmake/xmake.sh
+++ b/core/src/xmake/xmake.sh
@@ -70,6 +70,7 @@ target "xmake"
     add_files "sandbox/*.c"
     add_files "semver/*.c"
     add_files "string/*.c"
+    add_files "utf8/*.c"
     add_files "utils/*.c"
     add_files "tty/*.c"
     add_files "binutils/*.c"
diff --git a/tests/modules/utf8/test.lua b/tests/modules/utf8/test.lua
new file mode 100644
index 000000000..ccf9d0b3a
--- /dev/null
+++ b/tests/modules/utf8/test.lua
@@ -0,0 +1,64 @@
+
+function test_len(t)
+    t:are_equal(utf8.len("A"), 1)
+    t:are_equal(utf8.len("¢"), 1)
+    t:are_equal(utf8.len("€"), 1)
+    t:are_equal(utf8.len("𐍈"), 1)
+    t:are_equal(utf8.len("ab"), 2)
+    t:are_equal(utf8.len("A€B"), 3)
+    t:are_equal(utf8.len("你好"), 2)
+end
+
+function test_char(t)
+    t:are_equal(utf8.char(65), "A")
+    t:are_equal(utf8.char(0x20AC), "€")
+    t:are_equal(utf8.char(65, 66, 67), "ABC")
+end
+
+function test_codepoint(t)
+    t:are_equal(utf8.codepoint("A"), 65)
+    t:are_equal(utf8.codepoint("€"), 0x20AC)
+    local c1, c2, c3 = utf8.codepoint("ABC", 1, 3)
+    t:are_equal(c1, 65)
+    t:are_equal(c2, 66)
+    t:are_equal(c3, 67)
+    
+    -- test range
+    t:are_equal(utf8.codepoint("ABC", 2), 66)
+    t:are_equal(utf8.codepoint("ABC", 2, 2), 66)
+end
+
+function test_offset(t)
+    t:are_equal(utf8.offset("ABC", 1), 1)
+    t:are_equal(utf8.offset("ABC", 2), 2)
+    t:are_equal(utf8.offset("ABC", 4), 4)
+    t:are_equal(utf8.offset("ABC", 5), nil)
+    
+    -- "€" is 3 bytes (0xE2 0x82 0xAC)
+    t:are_equal(utf8.offset("€BC", 1), 1)
+    t:are_equal(utf8.offset("€BC", 2), 4)
+    t:are_equal(utf8.offset("€BC", 3), 5)
+
+    t:are_equal(utf8.offset("你好", 1), 1)
+    t:are_equal(utf8.offset("你好", 2), 4)
+    t:are_equal(utf8.offset("你好", 3), 7)
+end
+
+function test_codes(t)
+    local s = "A€"
+    local codes = {}
+    for p, c in utf8.codes(s) do
+        table.insert(codes, {p, c})
+    end
+    t:are_equal(#codes, 2)
+    t:are_equal(codes[1][1], 1)
+    t:are_equal(codes[1][2], 65)
+    -- "€" starts at 2? No, byte offset.
+    -- "A" is 1 byte. "€" starts at 2.
+    t:are_equal(codes[2][1], 2)
+    t:are_equal(codes[2][2], 0x20AC)
+end
+
+function test_charpattern(t)
+    t:require(utf8.charpattern)
+end
diff --git a/xmake/core/base/utf8.lua b/xmake/core/base/utf8.lua
new file mode 100644
index 000000000..d855b5fde
--- /dev/null
+++ b/xmake/core/base/utf8.lua
@@ -0,0 +1,34 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utf8.lua
+-- @desc        The utf8 module
+--              It provides basic support for UTF-8 encoding.
+--              It is compatible with Lua 5.3+ utf8 library.
+--
+-- @interface   utf8.len(s [, i [, j [, lax]]])
+-- @interface   utf8.char(...)
+-- @interface   utf8.codepoint(s [, i [, j]])
+-- @interface   utf8.offset(s, n [, i])
+-- @interface   utf8.codes(s [, lax])
+--
+
+-- define module: utf8
+local utf8 = utf8 or {}
+
+-- return module: utf8
+return utf8
diff --git a/xmake/core/sandbox/modules/interpreter/utf8.lua b/xmake/core/sandbox/modules/interpreter/utf8.lua
new file mode 100644
index 000000000..0b8ef72a4
--- /dev/null
+++ b/xmake/core/sandbox/modules/interpreter/utf8.lua
@@ -0,0 +1,22 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utf8.lua
+--
+
+-- load module
+return require("sandbox/modules/utf8")
diff --git a/xmake/core/sandbox/modules/utf8.lua b/xmake/core/sandbox/modules/utf8.lua
new file mode 100644
index 000000000..a8450dc87
--- /dev/null
+++ b/xmake/core/sandbox/modules/utf8.lua
@@ -0,0 +1,21 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utf8.lua
+--
+
+return require("base/utf8")