add io.convert

6 files changed, 280 insertions, 3 deletions

diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index d46e30a42..532be1adc 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -174,6 +174,7 @@ tb_int_t xm_io_file_rawfd(lua_State *lua);
 tb_int_t xm_io_file_write(lua_State *lua);
 tb_int_t xm_io_file_flush(lua_State *lua);
 tb_int_t xm_io_file_close(lua_State *lua);
+tb_int_t xm_io_file_convert(lua_State *lua);
 tb_int_t xm_io_file_isatty(lua_State *lua);
 
 // the io/filelock functions
@@ -480,6 +481,7 @@ static luaL_Reg const g_io_functions[] = {
     { "file_flush", xm_io_file_flush },
     { "file_isatty", xm_io_file_isatty },
     { "file_close", xm_io_file_close },
+    { "file_convert", xm_io_file_convert },
     { "file_rawfd", xm_io_file_rawfd },
     { "filelock_open", xm_io_filelock_open },
     { "filelock_lock", xm_io_filelock_lock },
diff --git a/core/src/xmake/io/file_convert.c b/core/src/xmake/io/file_convert.c
new file mode 100644
index 000000000..b1837c8d3
--- /dev/null
+++ b/core/src/xmake/io/file_convert.c
@@ -0,0 +1,224 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        file_convert.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * trace
+ */
+#define TB_TRACE_MODULE_NAME "file_convert"
+#define TB_TRACE_MODULE_DEBUG (0)
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "prefix.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * globals
+ */
+
+// the charset entry type
+typedef struct __xm_charset_entry_t {
+    // the charset type
+    tb_size_t type;
+
+    // the charset name
+    tb_char_t const *name;
+} xm_charset_entry_t, *xm_charset_entry_ref_t;
+
+// the charsets, @note: type & name is sorted
+static xm_charset_entry_t g_charsets[] = {
+    { TB_CHARSET_TYPE_ANSI, "ansi" },
+    { TB_CHARSET_TYPE_ASCII, "ascii" },
+    { TB_CHARSET_TYPE_GB2312, "gb2312" },
+    { TB_CHARSET_TYPE_GBK, "gbk" },
+    { TB_CHARSET_TYPE_ISO8859, "iso8859" },
+    { TB_CHARSET_TYPE_UCS2 | TB_CHARSET_TYPE_NE, "ucs2" },
+    { TB_CHARSET_TYPE_UCS4 | TB_CHARSET_TYPE_NE, "ucs4" },
+    { TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_NE, "utf16" },
+    { TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE, "utf16be" },
+    { TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_NE, "utf16bom" },
+    { TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE, "utf16le" },
+    { TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE, "utf16lebom" },
+    { TB_CHARSET_TYPE_UTF32 | TB_CHARSET_TYPE_NE, "utf32" },
+    { TB_CHARSET_TYPE_UTF32 | TB_CHARSET_TYPE_BE, "utf32be" },
+    { TB_CHARSET_TYPE_UTF32 | TB_CHARSET_TYPE_LE, "utf32le" },
+    { TB_CHARSET_TYPE_UTF8, "utf8" },
+    { TB_CHARSET_TYPE_UTF8, "utf8bom" },
+};
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * finder
+ */
+static tb_long_t xm_io_charset_comp_by_name(tb_iterator_ref_t iterator, tb_cpointer_t item, tb_cpointer_t name) {
+    return tb_stricmp(((xm_charset_entry_ref_t)item)->name, (tb_char_t const *)name);
+}
+static xm_charset_entry_ref_t xm_io_charset_find_by_name(tb_char_t const *name) {
+    // make iterator
+    tb_array_iterator_t array_iterator;
+    tb_iterator_ref_t   iterator =
+        tb_array_iterator_init_mem(&array_iterator, g_charsets, tb_arrayn(g_charsets), sizeof(xm_charset_entry_t));
+    tb_assert_and_check_return_val(iterator, tb_null);
+
+    // find it by the binary search
+    tb_size_t itor = tb_binary_find_all_if(iterator, xm_io_charset_comp_by_name, name);
+    if (itor != tb_iterator_tail(iterator)) {
+        return (xm_charset_entry_ref_t)tb_iterator_item(iterator, itor);
+    } else {
+        return tb_null;
+    }
+}
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+/* convert file
+ *
+ * @param srcpath   the srcpath
+ * @param dstpath   the dstpath
+ * @param ftype     the from-charset type, e.g. ascii, gb2312, gbk, ios8859, ucs2, ucs4, utf8, utf16, utf32
+ * @param ttype     the to-charset type
+ *
+ * @code
+ *      io.convert(srcpath, dstpath, "gbk", "utf8")
+ *      io.convert(srcpath, dstpath, "utf8", "gb2312")
+ * @endcode
+ */
+tb_int_t xm_io_file_convert(lua_State *lua) {
+    // check
+    tb_assert_and_check_return_val(lua, 0);
+
+#ifdef TB_CONFIG_MODULE_HAVE_CHARSET
+    // get arguments
+    tb_char_t const *srcpath = luaL_checkstring(lua, 1);
+    tb_char_t const *dstpath = luaL_checkstring(lua, 2);
+    tb_char_t const *fname   = luaL_checkstring(lua, 3);
+    tb_char_t const *tname   = luaL_checkstring(lua, 4);
+    tb_check_return_val(srcpath && dstpath && fname && tname, 0);
+
+    // get charset type
+    tb_size_t            ftype = TB_CHARSET_TYPE_UTF8;
+    tb_size_t            ttype = TB_CHARSET_TYPE_UTF8;
+    xm_charset_entry_ref_t fentry = xm_io_charset_find_by_name(fname);
+    if (fentry) ftype = fentry->type;
+    else {
+        lua_pushfstring(lua, "invalid charset: %s", fname);
+        lua_error(lua);
+    }
+    xm_charset_entry_ref_t tentry = xm_io_charset_find_by_name(tname);
+    if (tentry) ttype = tentry->type;
+    else {
+        lua_pushfstring(lua, "invalid charset: %s", tname);
+        lua_error(lua);
+    }
+
+    // done
+    tb_bool_t       ok      = tb_false;
+    tb_stream_ref_t istream = tb_null;
+    tb_stream_ref_t ostream = tb_null;
+    tb_stream_ref_t fstream = tb_null;
+    do {
+        // init istream
+        istream = tb_stream_init_from_file(srcpath, TB_FILE_MODE_RO);
+        tb_assert_and_check_break(istream);
+
+        // open istream
+        if (!tb_stream_open(istream)) break;
+
+        // skip bom
+        tb_size_t skip = 0;
+        tb_byte_t* bom_data = tb_null;
+        tb_long_t bom_size = tb_stream_peek(istream, &bom_data, 3);
+        if (bom_size >= 3 && ftype == TB_CHARSET_TYPE_UTF8 && bom_data[0] == 0xef && bom_data[1] == 0xbb && bom_data[2] == 0xbf) {
+            skip = 3;
+        } else if (bom_size >= 2 && (ftype & TB_CHARSET_TYPE_UTF16)) {
+             if (bom_data[0] == 0xff && bom_data[1] == 0xfe) skip = 2;
+             else if (bom_data[0] == 0xfe && bom_data[1] == 0xff) skip = 2;
+        }
+        if (skip > 0 && !tb_stream_skip(istream, skip)) break;
+
+        // init ostream
+        ostream = tb_stream_init_from_file(dstpath, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_TRUNC);
+        tb_assert_and_check_break(ostream);
+
+        // open ostream
+        if (!tb_stream_open(ostream)) break;
+
+        // write bom
+        if (!tb_strcmp(tname, "utf8bom")) {
+            static tb_byte_t const k_bom[] = {0xef, 0xbb, 0xbf};
+            if (!tb_stream_bwrit(ostream, k_bom, 3)) break;
+        } else if (!tb_strcmp(tname, "utf16lebom")) {
+            static tb_byte_t const k_bom[] = {0xff, 0xfe};
+            if (!tb_stream_bwrit(ostream, k_bom, 2)) break;
+        } else if (!tb_strcmp(tname, "utf16bom")) {
+#ifndef TB_WORDS_BIGENDIAN
+            static tb_byte_t const k_bom[] = {0xff, 0xfe};
+#else
+            static tb_byte_t const k_bom[] = {0xfe, 0xff};
+#endif
+            if (!tb_stream_bwrit(ostream, k_bom, 2)) break;
+        }
+
+        // init fstream
+        fstream = tb_stream_init_filter_from_charset(istream, ftype, ttype);
+        tb_assert_and_check_break(fstream);
+
+        // open fstream
+        if (!tb_stream_open(fstream)) break;
+
+        // transfer
+        tb_byte_t data[TB_STREAM_BLOCK_MAXN];
+        while (1) {
+            tb_long_t real = tb_stream_read(fstream, data, sizeof(data));
+            if (real > 0) {
+                if (!tb_stream_bwrit(ostream, data, real)) break;
+            } else if (real == 0) {
+                tb_long_t wait = tb_stream_wait(fstream, TB_STREAM_WAIT_READ, tb_stream_timeout(fstream));
+                if (wait <= 0) break;
+            } else break;
+        }
+
+        // ok
+        ok = tb_true;
+
+    } while (0);
+
+    // exit fstream
+    if (fstream) tb_stream_exit(fstream);
+    fstream = tb_null;
+
+    // exit istream
+    if (istream) tb_stream_exit(istream);
+    istream = tb_null;
+
+    // exit ostream
+    if (ostream) tb_stream_exit(ostream);
+    ostream = tb_null;
+
+    // ok?
+    lua_pushboolean(lua, ok);
+    return 1;
+#else
+    lua_pushboolean(lua, tb_false);
+    return 1;
+#endif
+}
diff --git a/tests/modules/io/test.lua b/tests/modules/io/test.lua
index e4c6f23e1..1776baba2 100644
--- a/tests/modules/io/test.lua
+++ b/tests/modules/io/test.lua
@@ -94,3 +94,23 @@ function test_write(t)
 
     os.tryrm("temp")
 end
+
+function test_convert(t)
+    local src = "files/utf8bom-lf-eleof"
+    local dst = "temp/convert_test.txt"
+    os.mkdir("temp")
+    
+    -- utf8 to gbk
+    io.convert(src, dst, {from = "utf8", to = "gbk"})
+    local content = io.readfile(dst, {encoding = "binary"})
+    t:are_equal(content, "123\\\n456\n789\n")
+    
+    -- gbk to utf8
+    local src_gbk = dst
+    local dst_utf8 = "temp/convert_test_utf8.txt"
+    io.convert(src_gbk, dst_utf8, {from = "gbk", to = "utf8"})
+    content = io.readfile(dst_utf8, {encoding = "binary"})
+    t:are_equal(content, "123\\\n456\n789\n")
+
+    os.tryrm("temp")
+end
diff --git a/xmake/core/base/io.lua b/xmake/core/base/io.lua
index f35c01ab3..b6bb79ecc 100644
--- a/xmake/core/base/io.lua
+++ b/xmake/core/base/io.lua
@@ -598,6 +598,27 @@ function io.close(file)
     return (file or io.stdout):close()
 end
 
+-- convert file encoding
+function io.convert(inputfile, outputfile, opt)
+    opt = opt or {}
+    inputfile = tostring(inputfile)
+    outputfile = tostring(outputfile)
+    local from = opt.from or "utf8"
+    local to = opt.to or "utf8"
+
+    -- try using c implementation first
+    if io.file_convert and io.file_convert(inputfile, outputfile, from, to) then
+        return true
+    end
+
+    -- fallback to lua implementation (slow but works)
+    local content, errors = io.readfile(inputfile, {encoding = from})
+    if not content then
+        return false, errors
+    end
+    return io.writefile(outputfile, content, {encoding = to})
+end
+
 -- save object the the given filepath
 function io.save(filepath, object, opt)
     opt = opt or {}
diff --git a/xmake/core/sandbox/modules/io.lua b/xmake/core/sandbox/modules/io.lua
index 1b15d4495..0cafaa5f2 100644
--- a/xmake/core/sandbox/modules/io.lua
+++ b/xmake/core/sandbox/modules/io.lua
@@ -201,6 +201,17 @@ function sandbox_io.insert(filepath, lineidx, text, opt)
     return data
 end
 
+-- convert file encoding
+function sandbox_io.convert(inputfile, outputfile, opt)
+    assert(inputfile and outputfile)
+    inputfile = vformat(inputfile)
+    outputfile = vformat(outputfile)
+    local ok, errors = io.convert(inputfile, outputfile, opt)
+    if not ok then
+        raise(errors)
+    end
+end
+
 -- get std file
 function sandbox_io.stdfile(filepath)
     assert(filepath)
diff --git a/xmake/modules/cli/iconv.lua b/xmake/modules/cli/iconv.lua
index cba86b500..efb26c864 100644
--- a/xmake/modules/cli/iconv.lua
+++ b/xmake/modules/cli/iconv.lua
@@ -47,7 +47,6 @@ function main(...)
     local from_code = args.from
     local to_code = args.to
 
-    -- read and write content
-    local content = io.readfile(input_file, {encoding = from_code})
-    io.writefile(output_file, content, {encoding = to_code})
+    -- convert encoding
+    io.convert(input_file, output_file, {from = from_code, to = to_code})
 end