add utf8.width

6 files changed, 200 insertions, 0 deletions

diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index 9ef4ec68f..146cdd17a 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -288,6 +288,7 @@ tb_int_t xm_utf8_codes(lua_State *lua);
 tb_int_t xm_utf8_sub(lua_State *lua);
 tb_int_t xm_utf8_lastof(lua_State *lua);
 tb_int_t xm_utf8_find(lua_State *lua);
+tb_int_t xm_utf8_width(lua_State *lua);
 
 // the string functions
 tb_int_t xm_string_trim(lua_State *lua);
@@ -603,6 +604,9 @@ static luaL_Reg const g_utf8_functions[] = {
     {"sub", xm_utf8_sub},
     {"lastof", xm_utf8_lastof},
     {"find", xm_utf8_find},
+    {"width", xm_utf8_width},
+    {"wcwidth", xm_utf8_width},
+    {"wcswidth", xm_utf8_width},
     {tb_null, tb_null}
 };
 
diff --git a/core/src/xmake/utf8/utf8.c b/core/src/xmake/utf8/utf8.c
index 3748e0975..168ffe60a 100644
--- a/core/src/xmake/utf8/utf8.c
+++ b/core/src/xmake/utf8/utf8.c
@@ -125,6 +125,122 @@ tb_long_t xm_utf8_charpos(tb_char_t const* s, tb_size_t len, tb_long_t byte_pos)
 /* //////////////////////////////////////////////////////////////////////////////////////
  * implementation interfaces
  */
+static struct { xm_utf8_int_t first; xm_utf8_int_t last; } const g_non_spacing[] = 
+{
+    {0x0300, 0x036F},   {0x0483, 0x0486},   {0x0488, 0x0489},
+    {0x0591, 0x05BD},   {0x05BF, 0x05BF},   {0x05C1, 0x05C2},
+    {0x05C4, 0x05C5},   {0x05C7, 0x05C7},   {0x0600, 0x0603},
+    {0x0610, 0x0615},   {0x064B, 0x065E},   {0x0670, 0x0670},
+    {0x06D6, 0x06E4},   {0x06E7, 0x06E8},   {0x06EA, 0x06ED},
+    {0x070F, 0x070F},   {0x0711, 0x0711},   {0x0730, 0x074A},
+    {0x07A6, 0x07B0},   {0x07EB, 0x07F3},   {0x0901, 0x0902},
+    {0x093C, 0x093C},   {0x0941, 0x0948},   {0x094D, 0x094D},
+    {0x0951, 0x0954},   {0x0962, 0x0963},   {0x0981, 0x0981},
+    {0x09BC, 0x09BC},   {0x09C1, 0x09C4},   {0x09CD, 0x09CD},
+    {0x09E2, 0x09E3},   {0x0A01, 0x0A02},   {0x0A3C, 0x0A3C},
+    {0x0A41, 0x0A42},   {0x0A47, 0x0A48},   {0x0A4B, 0x0A4D},
+    {0x0A70, 0x0A71},   {0x0A81, 0x0A82},   {0x0ABC, 0x0ABC},
+    {0x0AC1, 0x0AC5},   {0x0AC7, 0x0AC8},   {0x0ACD, 0x0ACD},
+    {0x0AE2, 0x0AE3},   {0x0B01, 0x0B01},   {0x0B3C, 0x0B3C},
+    {0x0B3F, 0x0B3F},   {0x0B41, 0x0B43},   {0x0B4D, 0x0B4D},
+    {0x0B56, 0x0B56},   {0x0B82, 0x0B82},   {0x0BC0, 0x0BC0},
+    {0x0BCD, 0x0BCD},   {0x0C3E, 0x0C40},   {0x0C46, 0x0C48},
+    {0x0C4A, 0x0C4D},   {0x0C55, 0x0C56},   {0x0CBC, 0x0CBC},
+    {0x0CBF, 0x0CBF},   {0x0CC6, 0x0CC6},   {0x0CCC, 0x0CCD},
+    {0x0CE2, 0x0CE3},   {0x0D41, 0x0D43},   {0x0D4D, 0x0D4D},
+    {0x0DCA, 0x0DCA},   {0x0DD2, 0x0DD4},   {0x0DD6, 0x0DD6},
+    {0x0E31, 0x0E31},   {0x0E34, 0x0E3A},   {0x0E47, 0x0E4E},
+    {0x0EB1, 0x0EB1},   {0x0EB4, 0x0EB9},   {0x0EBB, 0x0EBC},
+    {0x0EC8, 0x0ECD},   {0x0F18, 0x0F19},   {0x0F35, 0x0F35},
+    {0x0F37, 0x0F37},   {0x0F39, 0x0F39},   {0x0F71, 0x0F7E},
+    {0x0F80, 0x0F84},   {0x0F86, 0x0F87},   {0x0F90, 0x0F97},
+    {0x0F99, 0x0FBC},   {0x0FC6, 0x0FC6},   {0x102D, 0x1030},
+    {0x1032, 0x1032},   {0x1036, 0x1037},   {0x1039, 0x1039},
+    {0x1058, 0x1059},   {0x1160, 0x11FF},   {0x135F, 0x135F},
+    {0x1712, 0x1714},   {0x1732, 0x1734},   {0x1752, 0x1753},
+    {0x1772, 0x1773},   {0x17B4, 0x17B5},   {0x17B7, 0x17BD},
+    {0x17C6, 0x17C6},   {0x17C9, 0x17D3},   {0x17DD, 0x17DD},
+    {0x180B, 0x180D},   {0x18A9, 0x18A9},   {0x1920, 0x1922},
+    {0x1927, 0x1928},   {0x1932, 0x1932},   {0x1939, 0x193B},
+    {0x1A17, 0x1A18},   {0x1B00, 0x1B03},   {0x1B34, 0x1B34},
+    {0x1B36, 0x1B3A},   {0x1B3C, 0x1B3C},   {0x1B42, 0x1B42},
+    {0x1B6B, 0x1B73},   {0x1DC0, 0x1DCA},   {0x1DFE, 0x1DFF},
+    {0x200B, 0x200F},   {0x202A, 0x202E},   {0x2060, 0x2063},
+    {0x206A, 0x206F},   {0x20D0, 0x20EF},   {0x302A, 0x302F},
+    {0x3099, 0x309A},   {0xA806, 0xA806},   {0xA80B, 0xA80B},
+    {0xA825, 0xA826},   {0xFB1E, 0xFB1E},   {0xFE00, 0xFE0F},
+    {0xFE20, 0xFE23},   {0xFEFF, 0xFEFF},   {0xFFF9, 0xFFFB},
+    {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F},
+    {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x1D167, 0x1D169},
+    {0x1D173, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD},
+    {0x1D242, 0x1D244}, {0xE0001, 0xE0001}, {0xE0020, 0xE007F},
+    {0xE0100, 0xE01EF}
+};
+
+tb_long_t xm_utf8_charwidth(xm_utf8_int_t val) {
+
+    // test for 8-bit control characters
+    if (val == 0) return 0;
+    if (val < 32 || (val >= 0x7f && val < 0xa0)) {
+        if (val == 0x09) return 4; // TAB
+        if (val == 0x08) return -1; // BS
+        return 0; // other control chars
+    }
+
+    // binary search in table of non-spacing characters
+    tb_long_t min = 0;
+    tb_long_t max = tb_arrayn(g_non_spacing) - 1;
+    if (val >= g_non_spacing[0].first && val <= g_non_spacing[max].last) {
+        while (max >= min) {
+            tb_long_t mid = (min + max) / 2;
+            if (val > g_non_spacing[mid].last) {
+                min = mid + 1;
+            } else if (val < g_non_spacing[mid].first) {
+                max = mid - 1;
+            } else {
+                return 0;
+            }
+        }
+    }
+
+    if  (val >= 0x1100 && (val <= 0x115f ||  // Hangul Jamo init. consonants
+        val == 0x2329 || val == 0x232a ||
+        (val >= 0x2e80 && val <= 0xa4cf &&
+        val != 0x303f) ||                    // CJK ... Yi
+        (val >= 0xac00 && val <= 0xd7a3) ||  // Hangul Syllables
+        (val >= 0xf900 && val <= 0xfaff) ||  // CJK Compatibility Ideographs
+        (val >= 0xfe10 && val <= 0xfe19) ||  // Vertical forms
+        (val >= 0xfe30 && val <= 0xfe6f) ||  // CJK Compatibility Forms
+        (val >= 0xff00 && val <= 0xff60) ||  // Fullwidth Forms
+        (val >= 0xffe0 && val <= 0xffe6) ||
+        (val >= 0x20000 && val <= 0x2fffd) ||
+        (val >= 0x30000 && val <= 0x3fffd))) {
+        return 2;
+    }
+
+    return 1;
+}
+
+tb_long_t xm_utf8_strwidth(tb_char_t const* s, tb_size_t len) {
+    tb_assert_and_check_return_val(s, -1);
+    
+    tb_long_t width = 0;
+    tb_char_t const* p = s;
+    tb_char_t const* e = s + len;
+    while (p < e) {
+        xm_utf8_int_t val;
+        tb_char_t const* next = xm_utf8_decode(p, &val, tb_true);
+        if (next) {
+            tb_long_t w = xm_utf8_charwidth(val);
+            if (w < 0) return -1;
+            width += w;
+            p = next;
+        } else {
+            p++; // invalid byte, skip
+        }
+    }
+    return width;
+}
 
 tb_long_t xm_utf8_len_impl(tb_char_t const* s, tb_size_t len, tb_long_t posi, tb_long_t posj, tb_bool_t strict, tb_size_t* errpos) {
     tb_assert_and_check_return_val(s, -1);
diff --git a/core/src/xmake/utf8/utf8.h b/core/src/xmake/utf8/utf8.h
index f1d374af3..d6d884240 100644
--- a/core/src/xmake/utf8/utf8.h
+++ b/core/src/xmake/utf8/utf8.h
@@ -42,6 +42,8 @@ static __tb_inline__ tb_long_t xm_utf8_posrelat(tb_long_t pos, tb_size_t len) {
 tb_char_t const*    xm_utf8_decode(tb_char_t const* s, xm_utf8_int_t* val, tb_bool_t strict);
 tb_size_t           xm_utf8_encode(tb_char_t* s, xm_utf8_int_t val);
 tb_long_t           xm_utf8_charpos(tb_char_t const* s, tb_size_t len, tb_long_t byte_pos);
+tb_long_t           xm_utf8_charwidth(xm_utf8_int_t val);
+tb_long_t           xm_utf8_strwidth(tb_char_t const* s, tb_size_t len);
 
 /* //////////////////////////////////////////////////////////////////////////////////////
  * implementation interfaces
diff --git a/core/src/xmake/utf8/width.c b/core/src/xmake/utf8/width.c
new file mode 100644
index 000000000..615572462
--- /dev/null
+++ b/core/src/xmake/utf8/width.c
@@ -0,0 +1,45 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        width.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "utf8.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * interfaces
+ */
+
+/* utf8.width(char)
+ * utf8.width(str)
+ * utf8.width(codepoint)
+ */
+tb_int_t xm_utf8_width(lua_State* lua) {
+    if (lua_isnumber(lua, 1)) {
+        xm_utf8_int_t val = (xm_utf8_int_t)lua_tointeger(lua, 1);
+        lua_pushinteger(lua, xm_utf8_charwidth(val));
+    } else {
+        size_t len = 0;
+        tb_char_t const* s = luaL_checklstring(lua, 1, &len);
+        lua_pushinteger(lua, xm_utf8_strwidth(s, len));
+    }
+    return 1;
+}
diff --git a/tests/modules/utf8/test.lua b/tests/modules/utf8/test.lua
index af0c98997..df09604a4 100644
--- a/tests/modules/utf8/test.lua
+++ b/tests/modules/utf8/test.lua
@@ -142,3 +142,35 @@ function test_find(t)
     -- "A你好", "%w" -> matches "A".
     t:are_equal({utf8.find("A你好", "%w")}, {1, 1})
 end
+
+function test_width(t)
+    -- char/codepoint width
+    t:are_equal(utf8.width(string.byte("A")), 1)
+    t:are_equal(utf8.width(utf8.codepoint("€")), 1)
+    t:are_equal(utf8.width(utf8.codepoint("你")), 2)
+    t:are_equal(utf8.width(0), 0)
+    t:are_equal(utf8.width(0x09), 4) -- TAB
+
+    -- string width
+    t:are_equal(utf8.width("A"), 1)
+    t:are_equal(utf8.width("ABC"), 3)
+    t:are_equal(utf8.width("你好"), 4)
+    t:are_equal(utf8.width("A你好"), 5)
+    t:are_equal(utf8.width("A\tB"), 6) -- 1 + 4 + 1
+end
+
+function test_wcwidth(t)
+    -- char/codepoint width
+    t:are_equal(utf8.wcwidth(string.byte("A")), 1)
+    t:are_equal(utf8.wcwidth(utf8.codepoint("€")), 1)
+    t:are_equal(utf8.wcwidth(utf8.codepoint("你")), 2)
+end
+
+function test_wcswidth(t)
+    -- string width
+    t:are_equal(utf8.wcswidth("A"), 1)
+    t:are_equal(utf8.wcswidth("ABC"), 3)
+    t:are_equal(utf8.wcswidth("你好"), 4)
+    t:are_equal(utf8.wcswidth("A你好"), 5)
+    t:are_equal(utf8.wcswidth("A\tB"), 6)
+end
diff --git a/xmake/core/base/utf8.lua b/xmake/core/base/utf8.lua
index dfcfdfbf0..fefb3c94d 100644
--- a/xmake/core/base/utf8.lua
+++ b/xmake/core/base/utf8.lua
@@ -32,6 +32,7 @@ local utf8 = utf8 or {}
 -- @interface   utf8.sub(s, i [, j])
 -- @interface   utf8.lastof(s, pattern [, plain])
 -- @interface   utf8.find(s, pattern [, init [, plain]])
+-- @interface   utf8.width(s)
 --
 
 -- the char pattern