Merge pull request #5762 from xmake-io/bin2c

Use native implementation to improve bin2c speed

7 files changed, 257 insertions, 9 deletions

diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index 5fab7e7c1..8753607d4 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -316,6 +316,9 @@ tb_int_t xm_tty_term_mode(lua_State* lua);
 // the package functions
 tb_int_t xm_package_loadxmi(lua_State* lua);
 
+// the utils functions
+tb_int_t xm_utils_bin2c(lua_State* lua);
+
 #ifdef XM_CONFIG_API_HAVE_CURSES
 // register curses functions
 tb_int_t xm_lua_curses_register(lua_State* lua, tb_char_t const* module);
@@ -600,6 +603,13 @@ static luaL_Reg const g_package_functions[] =
 ,   { tb_null,          tb_null             }
 };
 
+// the utils functions
+static luaL_Reg const g_utils_functions[] =
+{
+    { "bin2c",          xm_utils_bin2c      }
+,   { tb_null,          tb_null             }
+};
+
 // the lua global instance for signal handler
 static lua_State* g_lua = tb_null;
 
@@ -1364,6 +1374,9 @@ xm_engine_ref_t xm_engine_init(tb_char_t const* name, xm_engine_lni_initalizer_c
         // bind package functions
         xm_lua_register(engine->lua, "package", g_package_functions);
 
+        // bind utils functions
+        xm_lua_register(engine->lua, "utils", g_utils_functions);
+
 #ifdef XM_CONFIG_API_HAVE_CURSES
         // bind curses
         xm_lua_curses_register(engine->lua, "curses");
diff --git a/core/src/xmake/utils/bin2c.c b/core/src/xmake/utils/bin2c.c
new file mode 100644
index 000000000..46da3eb38
--- /dev/null
+++ b/core/src/xmake/utils/bin2c.c
@@ -0,0 +1,165 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, TBOOX Open Source Group.
+ *
+ * @author      ruki
+ * @file        bin2c.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * trace
+ */
+#define TB_TRACE_MODULE_NAME                "bin2c"
+#define TB_TRACE_MODULE_DEBUG               (0)
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "prefix.h"
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * private implementation
+ */
+static __tb_inline__ tb_size_t xm_utils_bin2c_hex2str(tb_char_t str[5], tb_byte_t value)
+{
+    static tb_char_t const* digits_table = "0123456789ABCDEF";
+    str[0] = ' ';
+    str[1] = '0';
+    str[2] = 'x';
+    str[3] = digits_table[(value >> 4) & 15];
+    str[4] = digits_table[value & 15];
+    return 5;
+}
+
+static tb_bool_t xm_utils_bin2c_dump(tb_stream_ref_t istream, tb_stream_ref_t ostream, tb_int_t linewidth, tb_bool_t nozeroend)
+{
+    tb_bool_t first = tb_true;
+    tb_hong_t i = 0;
+    tb_hong_t left = 0;
+    tb_char_t line[4096];
+    tb_byte_t data[512];
+    tb_size_t linesize = 0;
+    tb_size_t need = 0;
+    tb_assert_and_check_return_val(linewidth < sizeof(data), tb_false);
+    while (!tb_stream_beof(istream))
+    {
+        linesize = 0;
+        left = tb_stream_left(istream);
+        need = (tb_size_t)tb_min(left, linewidth);
+        if (need)
+        {
+            if (!tb_stream_bread(istream, data, need))
+                break;
+
+            if (!nozeroend && tb_stream_beof(istream))
+            {
+                tb_assert_and_check_break(need + 1 < sizeof(data));
+                data[need++] = '\0';
+            }
+
+            tb_assert_and_check_break(linesize + 6 * need < sizeof(line));
+
+            i = 0;
+            if (first)
+            {
+                first = tb_false;
+                line[linesize++] = ' ';
+            }
+            else line[linesize++] = ',';
+            linesize += xm_utils_bin2c_hex2str(line + linesize, data[i]);
+
+            for (i = 1; i < need; i++)
+            {
+                line[linesize++] = ',';
+                linesize += xm_utils_bin2c_hex2str(line + linesize, data[i]);
+            }
+            tb_assert_and_check_break(i == need && linesize && linesize < sizeof(line));
+
+            if (tb_stream_bwrit_line(ostream, line, linesize) < 0)
+                break;
+        }
+    }
+
+    return tb_stream_beof(istream);
+}
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+/* generate c/c++ code from the binary file
+ *
+ * local ok, errors = utils.bin2c(binaryfile, outputfile, linewidth, nozeroend)
+ */
+tb_int_t xm_utils_bin2c(lua_State* lua)
+{
+    // check
+    tb_assert_and_check_return_val(lua, 0);
+
+    // get the binaryfile
+    tb_char_t const* binaryfile = luaL_checkstring(lua, 1);
+    tb_check_return_val(binaryfile, 0);
+
+    // get the outputfile
+    tb_char_t const* outputfile = luaL_checkstring(lua, 2);
+    tb_check_return_val(outputfile, 0);
+
+    // get line width
+    tb_int_t linewidth = (tb_int_t)lua_tointeger(lua, 3);
+
+    // no zero end?
+    tb_bool_t nozeroend = (tb_bool_t)lua_toboolean(lua, 4);
+
+    // do dump
+    tb_bool_t ok = tb_false;
+    tb_stream_ref_t istream = tb_stream_init_from_file(binaryfile, TB_FILE_MODE_RO);
+    tb_stream_ref_t ostream = tb_stream_init_from_file(outputfile, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_TRUNC);
+    do
+    {
+        if (!tb_stream_open(istream))
+        {
+            lua_pushboolean(lua, tb_false);
+            lua_pushfstring(lua, "bin2c: open %s failed", binaryfile);
+            break;
+        }
+
+        if (!tb_stream_open(ostream))
+        {
+            lua_pushboolean(lua, tb_false);
+            lua_pushfstring(lua, "bin2c: open %s failed", outputfile);
+            break;
+        }
+
+        if (!xm_utils_bin2c_dump(istream, ostream, linewidth, nozeroend))
+        {
+            lua_pushboolean(lua, tb_false);
+            lua_pushfstring(lua, "bin2c: dump data failed");
+            break;
+        }
+
+        ok = tb_true;
+        lua_pushboolean(lua, ok);
+
+    } while (0);
+
+    if (istream) tb_stream_clos(istream);
+    istream = tb_null;
+
+    if (ostream) tb_stream_clos(ostream);
+    ostream = tb_null;
+
+    return ok? 1 : 2;
+}
diff --git a/core/src/xmake/utils/prefix.h b/core/src/xmake/utils/prefix.h
new file mode 100644
index 000000000..e56baedf7
--- /dev/null
+++ b/core/src/xmake/utils/prefix.h
@@ -0,0 +1,31 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, TBOOX Open Source Group.
+ *
+ * @author      ruki
+ * @file        prefix.h
+ *
+ */
+#ifndef XM_UTILS_PREFIX_H
+#define XM_UTILS_PREFIX_H
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "../prefix.h"
+
+#endif
+
+
diff --git a/core/src/xmake/xmake.sh b/core/src/xmake/xmake.sh
index 62af5b2fa..8bcc60632 100755
--- a/core/src/xmake/xmake.sh
+++ b/core/src/xmake/xmake.sh
@@ -71,6 +71,7 @@ target "xmake"
     add_files "semver/*.c"
     add_files "string/*.c"
     add_files "tty/*.c"
+    add_files "utils/*.c"
     if is_plat "mingw"; then
         add_files "winos/*.c"
     fi
diff --git a/xmake/core/base/utils.lua b/xmake/core/base/utils.lua
index 5125d224e..99add99bc 100644
--- a/xmake/core/base/utils.lua
+++ b/xmake/core/base/utils.lua
@@ -30,6 +30,9 @@ local io     = require("base/io")
 local dump   = require("base/dump")
 local text   = require("base/text")
 
+-- save original interfaces
+utils._bin2c = utils._bin2c or utils.bin2c
+
 -- dump values
 function utils.dump(...)
     if option.get("quiet") then
@@ -318,5 +321,11 @@ function utils.vtable(data, opt)
     utils.vprintf(text.table(data, opt))
 end
 
+-- generate c/c++ code from the binary file
+function utils.bin2c(binaryfile, outputfile, opt)
+    opt = opt or {}
+    return utils._bin2c(binaryfile, outputfile, opt.linewidth or 32, opt.nozeroend or false)
+end
+
 -- return module
 return utils
diff --git a/xmake/core/sandbox/modules/utils.lua b/xmake/core/sandbox/modules/utils.lua
index 42660d36e..9614e544b 100644
--- a/xmake/core/sandbox/modules/utils.lua
+++ b/xmake/core/sandbox/modules/utils.lua
@@ -148,6 +148,16 @@ function sandbox_utils.assert(value, format, ...)
     return value
 end
 
+-- generate c/c++ code from the binary file
+if utils._bin2c then
+    function sandbox_utils.bin2c(binaryfile, outputfile, opt)
+        local ok, errors = utils.bin2c(binaryfile, outputfile, opt)
+        if not ok then
+            os.raise(errors)
+        end
+    end
+end
+
 -- return module
 return sandbox_utils
 
diff --git a/xmake/modules/private/utils/bin2c.lua b/xmake/modules/private/utils/bin2c.lua
index 6f18833fc..29c89338e 100644
--- a/xmake/modules/private/utils/bin2c.lua
+++ b/xmake/modules/private/utils/bin2c.lua
@@ -35,7 +35,7 @@ function _do_dump(binarydata, outputfile, opt)
     local p = 0
     local e = binarydata:size()
     local line = nil
-    local linewidth = opt.linewidth or 0x20
+    local linewidth = opt.linewidth or 32
     local first = true
     while p < e do
         line = ""
@@ -81,22 +81,41 @@ function _do_bin2c(binarypath, outputpath, opt)
     -- trace
     print("generating code data file from %s ..", binarypath)
 
+    -- optimize the default linewidth for reading large file
+    if not opt.linewidth then
+        local filesize = os.filesize(binarypath)
+        if filesize > 1024 * 1024 * 1024 then
+            opt.linewidth = 512
+        elseif filesize > 100 * 1024 * 1024 then
+            opt.linewidth = 256
+        elseif filesize > 10 * 1024 * 1024 then
+            opt.linewidth = 128
+        elseif filesize > 1024 * 1024 then
+            opt.linewidth = 64
+        else
+            opt.linewidth = 32
+        end
+    end
+
     -- do dump
-    local binarydata = bytes(io.readfile(binarypath, {encoding = "binary"}))
-    local outputfile = io.open(outputpath, 'w')
-    if outputfile then
-        if not opt.nozeroend then
-            binarydata = binarydata .. bytes('\0')
+    if utils.bin2c then
+        utils.bin2c(binarypath, outputpath, opt)
+    else
+        local binarydata = bytes(io.readfile(binarypath, {encoding = "binary"}))
+        local outputfile = io.open(outputpath, 'w')
+        if outputfile then
+            if not opt.nozeroend then
+                binarydata = binarydata .. bytes('\0')
+            end
+            _do_dump(binarydata, outputfile, opt)
+            outputfile:close()
         end
-        _do_dump(binarydata, outputfile, opt)
-        outputfile:close()
     end
 
     -- trace
     cprint("${bright}%s generated!", outputpath)
 end
 
--- main entry
 function main(...)
 
     -- parse arguments