Merge pull request #4918 from xmake-io/modules

Add native modules support

28 files changed, 487 insertions, 155 deletions

diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml
index 896c585b5..5cb8e0edf 100644
--- a/.github/workflows/freebsd.yml
+++ b/.github/workflows/freebsd.yml
@@ -25,7 +25,7 @@ jobs:
           usesh: true
           mem: 4096
           copyback: false
-          prepare: pkg install -y curl unzip gmake llvm gsed bash perl5
+          prepare: pkg install -y git curl unzip gmake llvm gsed bash perl5
           run: |
             ./configure
             gmake -j4
diff --git a/.github/workflows/ubuntu_arm.yml b/.github/workflows/ubuntu_arm.yml
index a20c673ae..ad8a6f229 100644
--- a/.github/workflows/ubuntu_arm.yml
+++ b/.github/workflows/ubuntu_arm.yml
@@ -26,7 +26,7 @@ jobs:
           args: >
             bash -c
             "uname -a &&
-            apt update &&
+            apt update && apt install -y unzip &&
             cd /github/workspace &&
             ./scripts/get.sh __local__ &&
             source ~/.xmake/profile &&
diff --git a/.github/workflows/ubuntu_arm64.yml b/.github/workflows/ubuntu_arm64.yml
index 05d6e6f00..c2b6d4370 100644
--- a/.github/workflows/ubuntu_arm64.yml
+++ b/.github/workflows/ubuntu_arm64.yml
@@ -26,7 +26,7 @@ jobs:
           args: >
             bash -c
             "uname -a &&
-            apt update &&
+            apt update && apt install -y unzip &&
             cd /github/workspace &&
             ./scripts/get.sh __local__ &&
             source ~/.xmake/profile &&
diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index cbe1ed4fc..827d46d2e 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -933,11 +933,39 @@ static tb_void_t xm_engine_init_host(xm_engine_t* engine)
     lua_setglobal(engine->lua, "_SUBHOST");
 }
 
+static __tb_inline__ tb_char_t const* xm_engine_xmake_arch()
+{
+    tb_char_t const* arch = tb_null;
+#if defined(TB_CONFIG_OS_WINDOWS) && !defined(TB_COMPILER_LIKE_UNIX)
+#   if defined(TB_ARCH_x64)
+    arch = "x64";
+#   elif defined(TB_ARCH_ARM64)
+    arch = "arm64";
+#   elif defined(TB_ARCH_ARM)
+    arch = "arm";
+#   else
+    arch = "x86";
+#   endif
+#elif defined(TB_ARCH_x64)
+    arch = "x86_64";
+#elif defined(TB_ARCH_x86)
+    arch = "i386";
+#else
+    arch = TB_ARCH_STRING;
+#endif
+    return arch;
+}
+
 static tb_void_t xm_engine_init_arch(xm_engine_t* engine)
 {
     // check
     tb_assert_and_check_return(engine && engine->lua);
 
+    // init xmake arch
+    tb_char_t const* xmakearch = xm_engine_xmake_arch();
+    lua_pushstring(engine->lua, xmakearch);
+    lua_setglobal(engine->lua, "_XMAKE_ARCH");
+
     // init system architecture
     tb_char_t const* sysarch = tb_null;
 #if defined(TB_CONFIG_OS_WINDOWS) && !defined(TB_COMPILER_LIKE_UNIX)
@@ -972,27 +1000,8 @@ static tb_void_t xm_engine_init_arch(xm_engine_t* engine)
     default:
         break;
     }
-
-    // get arch from compiler
-    if (!sysarch)
-    {
-#   if defined(TB_ARCH_x64)
-        sysarch = "x64";
-#   elif defined(TB_ARCH_ARM64)
-        sysarch = "arm64";
-#   elif defined(TB_ARCH_ARM)
-        sysarch = "arm";
-#   else
-        sysarch = "x86";
-#   endif
-    }
-#elif defined(TB_ARCH_x64)
-    sysarch = "x86_64";
-#elif defined(TB_ARCH_x86)
-    sysarch = "i386";
-#else
-    sysarch = TB_ARCH_STRING;
 #endif
+    if (!sysarch) sysarch = xmakearch;
     lua_pushstring(engine->lua, sysarch);
     lua_setglobal(engine->lua, "_ARCH");
 
diff --git a/tests/projects/other/autogen_module/.gitignore b/tests/projects/other/autogen_module/.gitignore
new file mode 100644
index 000000000..152105761
--- /dev/null
+++ b/tests/projects/other/autogen_module/.gitignore
@@ -0,0 +1,8 @@
+# Xmake cache
+.xmake/
+build/
+
+# MacOS Cache
+.DS_Store
+
+
diff --git a/tests/projects/other/autogen_module/modules/autogen/.gitignore b/tests/projects/other/autogen_module/modules/autogen/.gitignore
new file mode 100644
index 000000000..152105761
--- /dev/null
+++ b/tests/projects/other/autogen_module/modules/autogen/.gitignore
@@ -0,0 +1,8 @@
+# Xmake cache
+.xmake/
+build/
+
+# MacOS Cache
+.DS_Store
+
+
diff --git a/tests/projects/other/autogen_module/modules/autogen/src/main.cpp b/tests/projects/other/autogen_module/modules/autogen/src/main.cpp
new file mode 100644
index 000000000..c5150cdbd
--- /dev/null
+++ b/tests/projects/other/autogen_module/modules/autogen/src/main.cpp
@@ -0,0 +1,29 @@
+#include <iostream>
+#include <fstream>
+#include <vector>
+
+using namespace std;
+
+int main(int argc, char** argv) {
+    ifstream src_file(argv[1], ios::in | ios::binary);
+    if (!src_file) {
+        return 1;
+    }
+    vector<char> buffer(istreambuf_iterator<char>(src_file), {});
+    src_file.close();
+
+    ofstream dst_file(argv[2], ios::out);
+    if (!dst_file) {
+        return 1;
+    }
+
+    dst_file << "unsigned char g_codegen_data[] = {";
+    for (auto byte : buffer) {
+        dst_file << "0x" << hex << (int)(unsigned char)byte << ",";
+    }
+    dst_file << "0};" << endl;
+    dst_file.close();
+    return 0;
+}
+
+
diff --git a/tests/projects/other/autogen_module/modules/autogen/xmake.lua b/tests/projects/other/autogen_module/modules/autogen/xmake.lua
new file mode 100644
index 000000000..677da3e2d
--- /dev/null
+++ b/tests/projects/other/autogen_module/modules/autogen/xmake.lua
@@ -0,0 +1,6 @@
+add_rules("mode.debug", "mode.release")
+
+target("generate")
+    add_rules("module.binary")
+    add_files("src/*.cpp")
+    set_languages("c++11")
diff --git a/tests/projects/other/autogen_module/src/data.in b/tests/projects/other/autogen_module/src/data.in
new file mode 100644
index 000000000..a04238969
--- /dev/null
+++ b/tests/projects/other/autogen_module/src/data.in
@@ -0,0 +1 @@
+hello world!
diff --git a/tests/projects/other/autogen_module/src/main.cpp b/tests/projects/other/autogen_module/src/main.cpp
new file mode 100644
index 000000000..3241308f7
--- /dev/null
+++ b/tests/projects/other/autogen_module/src/main.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+extern unsigned char g_codegen_data[];
+
+int main(int argc, char** argv) {
+    cout << (const char*)g_codegen_data << endl;
+    return 0;
+}
diff --git a/tests/projects/other/autogen_module/test.lua b/tests/projects/other/autogen_module/test.lua
new file mode 100644
index 000000000..b57362078
--- /dev/null
+++ b/tests/projects/other/autogen_module/test.lua
@@ -0,0 +1,3 @@
+function main(t)
+    t:build()
+end
diff --git a/tests/projects/other/autogen_module/xmake.lua b/tests/projects/other/autogen_module/xmake.lua
new file mode 100644
index 000000000..c5b194d2e
--- /dev/null
+++ b/tests/projects/other/autogen_module/xmake.lua
@@ -0,0 +1,32 @@
+add_rules("mode.debug", "mode.release")
+
+add_moduledirs("modules")
+
+rule("autogen")
+    set_extensions(".in")
+    before_build_file(function (target, sourcefile, opt)
+        import("utils.progress")
+        import("core.project.depend")
+        import("core.tool.compiler")
+        import("autogen")
+
+        local sourcefile_cx = path.join(target:autogendir(), "rules", "autogen", path.basename(sourcefile) .. ".cpp")
+        local objectfile = target:objectfile(sourcefile_cx)
+        table.insert(target:objectfiles(), objectfile)
+
+        depend.on_changed(function ()
+            progress.show(opt.progress, "${color.build.object}compiling.autogen %s", sourcefile)
+            os.mkdir(path.directory(sourcefile_cx))
+            autogen.generate(sourcefile, sourcefile_cx)
+            compiler.compile(sourcefile_cx, objectfile, {target = target})
+        end, {dependfile = target:dependfile(objectfile),
+              files = sourcefile,
+              changed = target:is_rebuilt()})
+    end)
+
+target("test")
+    set_kind("binary")
+    add_rules("autogen")
+    add_files("src/main.cpp")
+    add_files("src/*.in")
+
diff --git a/tests/projects/other/native_module/.gitignore b/tests/projects/other/native_module/.gitignore
new file mode 100644
index 000000000..152105761
--- /dev/null
+++ b/tests/projects/other/native_module/.gitignore
@@ -0,0 +1,8 @@
+# Xmake cache
+.xmake/
+build/
+
+# MacOS Cache
+.DS_Store
+
+
diff --git a/tests/projects/other/native_module/modules/binary/bar/.gitignore b/tests/projects/other/native_module/modules/binary/bar/.gitignore
new file mode 100644
index 000000000..152105761
--- /dev/null
+++ b/tests/projects/other/native_module/modules/binary/bar/.gitignore
@@ -0,0 +1,8 @@
+# Xmake cache
+.xmake/
+build/
+
+# MacOS Cache
+.DS_Store
+
+
diff --git a/tests/projects/other/native_module/modules/binary/bar/src/add.cpp b/tests/projects/other/native_module/modules/binary/bar/src/add.cpp
new file mode 100644
index 000000000..e19d07e12
--- /dev/null
+++ b/tests/projects/other/native_module/modules/binary/bar/src/add.cpp
@@ -0,0 +1,9 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char** argv) {
+    int a = atoi(argv[1]);
+    int b = atoi(argv[2]);
+    printf("%d", a + b);
+    return 0;
+}
diff --git a/tests/projects/other/native_module/modules/binary/bar/src/sub.cpp b/tests/projects/other/native_module/modules/binary/bar/src/sub.cpp
new file mode 100644
index 000000000..980289d1a
--- /dev/null
+++ b/tests/projects/other/native_module/modules/binary/bar/src/sub.cpp
@@ -0,0 +1,10 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char** argv) {
+    int a = atoi(argv[1]);
+    int b = atoi(argv[2]);
+    printf("%d", a - b);
+    return 0;
+}
+
diff --git a/tests/projects/other/native_module/modules/binary/bar/xmake.lua b/tests/projects/other/native_module/modules/binary/bar/xmake.lua
new file mode 100644
index 000000000..add3c6e97
--- /dev/null
+++ b/tests/projects/other/native_module/modules/binary/bar/xmake.lua
@@ -0,0 +1,10 @@
+add_rules("mode.debug", "mode.release")
+
+target("add")
+    add_rules("module.binary")
+    add_files("src/add.cpp")
+
+target("sub")
+    add_rules("module.binary")
+    add_files("src/sub.cpp")
+
diff --git a/tests/projects/other/native_module/modules/shared/foo/src/foo.c b/tests/projects/other/native_module/modules/shared/foo/src/foo.c
new file mode 100644
index 000000000..3024f9380
--- /dev/null
+++ b/tests/projects/other/native_module/modules/shared/foo/src/foo.c
@@ -0,0 +1,29 @@
+#include <stdlib.h>
+#include <lua.h>
+#include <lauxlib.h>
+
+static int add(lua_State* lua) {
+    int a = lua_tointeger(lua, 1);
+    int b = lua_tointeger(lua, 2);
+    lua_pushinteger(lua, a + b);
+    return 1;
+}
+
+static int sub(lua_State* lua) {
+    int a = lua_tointeger(lua, 1);
+    int b = lua_tointeger(lua, 2);
+    lua_pushinteger(lua, a - b);
+    return 1;
+}
+
+static const luaL_Reg g_funcs[] = {
+    {"add", add},
+    {"sub", sub},
+    {NULL, NULL}
+};
+
+int luaopen_foo(lua_State* lua) {
+    lua_newtable(lua);
+    luaL_setfuncs(lua, g_funcs, 0);
+    return 1;
+}
diff --git a/tests/projects/other/native_module/modules/shared/foo/xmake.lua b/tests/projects/other/native_module/modules/shared/foo/xmake.lua
new file mode 100644
index 000000000..ad4590ffa
--- /dev/null
+++ b/tests/projects/other/native_module/modules/shared/foo/xmake.lua
@@ -0,0 +1,9 @@
+add_rules("mode.debug", "mode.release")
+
+add_requires("lua 5.4")
+
+target("foo")
+    add_rules("module.shared")
+    add_files("src/foo.c")
+    add_packages("lua")
+
diff --git a/tests/projects/other/native_module/src/main.cpp b/tests/projects/other/native_module/src/main.cpp
new file mode 100644
index 000000000..7c775d288
--- /dev/null
+++ b/tests/projects/other/native_module/src/main.cpp
@@ -0,0 +1,6 @@
+#include <iostream>
+
+int main(int argc, char** argv) {
+    std::cout << "hello world!" << std::endl;
+    return 0;
+}
diff --git a/tests/projects/other/native_module/test.lua b/tests/projects/other/native_module/test.lua
new file mode 100644
index 000000000..0c2735866
--- /dev/null
+++ b/tests/projects/other/native_module/test.lua
@@ -0,0 +1,5 @@
+function main(t)
+    if not xmake.luajit() then
+        t:build()
+    end
+end
diff --git a/tests/projects/other/native_module/xmake.lua b/tests/projects/other/native_module/xmake.lua
new file mode 100644
index 000000000..396921051
--- /dev/null
+++ b/tests/projects/other/native_module/xmake.lua
@@ -0,0 +1,16 @@
+add_rules("mode.debug", "mode.release")
+
+add_moduledirs("modules")
+
+target("test")
+    set_kind("binary")
+    add_files("src/*.cpp")
+    on_config(function (target)
+        import("shared.foo")
+        import("binary.bar")
+        print("shared: 1 + 1 = %d", foo.add(1, 1))
+        print("shared: 1 - 1 = %d", foo.sub(1, 1))
+        print("binary: 1 + 1 = %s", bar.add(1, 1))
+        print("binary: 1 - 1 = %s", bar.sub(1, 1))
+    end)
+
diff --git a/xmake/core/_xmake_main.lua b/xmake/core/_xmake_main.lua
index 6dba86db5..38f7df73d 100644
--- a/xmake/core/_xmake_main.lua
+++ b/xmake/core/_xmake_main.lua
@@ -26,6 +26,7 @@ xmake._HOST             = _HOST
 xmake._ARCH             = _ARCH
 xmake._SUBHOST          = _SUBHOST
 xmake._SUBARCH          = _SUBARCH
+xmake._XMAKE_ARCH       = _XMAKE_ARCH
 xmake._VERSION          = _VERSION
 xmake._VERSION_SHORT    = _VERSION_SHORT
 xmake._PROGRAM_DIR      = _PROGRAM_DIR
diff --git a/xmake/core/base/xmake.lua b/xmake/core/base/xmake.lua
index e892a5a04..dad96dc06 100644
--- a/xmake/core/base/xmake.lua
+++ b/xmake/core/base/xmake.lua
@@ -37,6 +37,11 @@ function xmake.version()
     return xmake._VERSION_CACHE or nil
 end
 
+-- get the xmake binary architecture
+function xmake.arch()
+    return xmake._XMAKE_ARCH
+end
+
 -- get the git branch of xmake version, e.g. build: {"dev", "d4cff6e11"}
 function xmake.branch()
     return xmake.version():build()[1]
diff --git a/xmake/core/sandbox/modules/import/core/sandbox/module.lua b/xmake/core/sandbox/modules/import/core/sandbox/module.lua
index 1c2dc6fbc..9678512de 100644
--- a/xmake/core/sandbox/modules/import/core/sandbox/module.lua
+++ b/xmake/core/sandbox/modules/import/core/sandbox/module.lua
@@ -24,14 +24,23 @@ local core_sandbox_module = core_sandbox_module or {}
 -- load modules
 local os        = require("base/os")
 local path      = require("base/path")
+local hash      = require("base/hash")
 local utils     = require("base/utils")
 local table     = require("base/table")
 local string    = require("base/string")
+local option    = require("base/option")
 local global    = require("base/global")
+local config    = require("project/config")
 local memcache  = require("cache/memcache")
 local sandbox   = require("sandbox/sandbox")
 local raise     = require("sandbox/modules/raise")
 
+-- the module kinds
+local MODULE_KIND_LUAFILE = 1
+local MODULE_KIND_LUADIR  = 2
+local MODULE_KIND_BINARY  = 3
+local MODULE_KIND_SHARED  = 4
+
 -- get module path from name
 function core_sandbox_module._modulepath(name)
 
@@ -60,8 +69,6 @@ end
 
 -- load module from file
 function core_sandbox_module._loadfile(filepath, instance)
-
-    -- check
     assert(filepath)
 
     -- load module script
@@ -84,8 +91,6 @@ function core_sandbox_module._loadfile(filepath, instance)
         if not result then
             return nil, errors
         end
-
-        -- ok
         return result, instance:script()
     end
 
@@ -94,174 +99,245 @@ function core_sandbox_module._loadfile(filepath, instance)
     if not ok then
         return nil, result
     end
-
-    -- ok?
     return result, script
 end
 
 -- find module
 function core_sandbox_module._find(dir, name)
-
-    -- check
     assert(dir and name)
 
-    -- get module path
-    name = core_sandbox_module._modulepath(name)
-    assert(name)
+    -- get module subpath
+    local module_subpath = core_sandbox_module._modulepath(name)
+    assert(module_subpath)
 
-    -- get module key
-    local key = path.join(dir, name)
+    -- get module full path
+    local module_fullpath = path.join(dir, module_subpath)
 
-    -- the single module?
-    if os.isfile(key .. ".lua") then
-        return path.normalize(path.absolute(key)), false
-    -- modules?
-    elseif os.isdir(key) then
-        return path.normalize(path.absolute(key)), true
+    -- single lua module?
+    local modulekey = path.normalize(path.absolute(module_fullpath))
+    if os.isfile(module_fullpath .. ".lua") then
+        return modulekey, MODULE_KIND_LUAFILE
+    elseif os.isdir(module_fullpath) then
+        local module_projectfile = path.join(module_fullpath, "xmake.lua")
+        if os.isfile(module_projectfile) then -- native module? e.g. binary/shared modules
+            local content = io.readfile(module_projectfile)
+            local kind = content:match("add_rules%(\"module.(.-)\"%)")
+            if kind == "binary" then
+                return modulekey, MODULE_KIND_BINARY
+            elseif kind == "shared" then
+                return modulekey, MODULE_KIND_SHARED
+            end
+        else
+            -- module directory
+            return modulekey, MODULE_KIND_LUADIR
+        end
     end
 end
 
--- load module
-function core_sandbox_module._load(dir, name, instance, module)
-
-    -- check
-    assert(dir and name)
+-- load module from the single script file
+function core_sandbox_module._load_from_scriptfile(module_fullpath, opt)
+    assert(not opt.module)
+    return core_sandbox_module._loadfile(module_fullpath .. ".lua", opt.instance)
+end
 
-    -- get module path
-    name = core_sandbox_module._modulepath(name)
-    assert(name)
+-- load module from the script directory
+function core_sandbox_module._load_from_scriptdir(module_fullpath, opt)
+    local script
+    local module = opt.module
+    local modulefiles = os.files(path.join(module_fullpath, "**.lua"))
+    if modulefiles then
+        for _, modulefile in ipairs(modulefiles) do
+            local result, errors = core_sandbox_module._loadfile(modulefile, opt.instance)
+            if not result then
+                return nil, errors
+            end
 
-    -- load the single module?
-    local script = nil
-    if os.isfile(path.join(dir, name .. ".lua")) then
+            -- bind main entry
+            if type(result) == "table" and result.main then
+                setmetatable(result, { __call = function (_, ...) return result.main(...) end})
+            end
 
-        -- check
-        assert(not module)
+            -- get the module path
+            local modulepath = path.relative(modulefile, module_fullpath)
+            if not modulepath then
+                return nil, string.format("cannot get the path for module: %s", module_subpath)
+            end
+            module = module or {}
+            script = errors
 
-        -- load module
-        local result, errors = core_sandbox_module._loadfile(path.join(dir, name .. ".lua"), instance)
-        if not result then
-            return nil, errors
+            -- save module
+            local scope = module
+            for _, modulename in ipairs(path.split(modulepath)) do
+                local pos = modulename:find(".lua", 1, true)
+                if pos then
+                    modulename = modulename:sub(1, pos - 1)
+                    assert(modulename)
+                    scope[modulename] = result
+                else
+                    -- enter submodule
+                    scope[modulename] = scope[modulename] or {}
+                    scope = scope[modulename]
+                end
+            end
         end
+    end
+    return module, script
+end
 
-        -- save module
-        module = result
-
-        -- save script
-        script = errors
-
-    -- load modules
-    elseif os.isdir(path.join(dir, name)) then
-
-        -- get modulefiles
-        local moduleroot = path.join(path.join(dir, name))
-        local modulefiles = os.match(path.join(moduleroot, "**.lua"))
-        if modulefiles then
-            for _, modulefile in ipairs(modulefiles) do
+-- add some builtin global options from the parent xmake
+function core_sandbox_module._add_builtin_argv(argv, projectdir)
+    table.insert(argv, "-P")
+    table.insert(argv, projectdir)
+    for _, name in ipairs({"diagnosis", "verbose", "quiet", "yes", "confirm", "root"}) do
+        local value = option.get(name)
+        if type(value) == "boolean" then
+            table.insert(argv, "--" .. name)
+        elseif value ~= nil then
+            table.insert(argv, "--" .. name .. "=" .. value)
+        end
+    end
+end
 
-                -- load module
-                local result, errors = core_sandbox_module._loadfile(modulefile, instance)
-                if not result then
-                    return nil, errors
-                end
+-- build module
+function core_sandbox_module._build_module(module_fullpath)
+    local projectdir = path.normalize(path.absolute(module_fullpath))
+    local programdir = path.normalize(os.programdir())
+    local modulehash = hash.uuid4(projectdir):split("-", {plain = true})[1]:lower()
+    local buildir
+    if projectdir:startswith(programdir) then
+        buildir = path.join(global.directory(), "cache", "modules", modulehash)
+    elseif os.isfile(os.projectfile()) then
+        buildir = path.join(config.directory(), "cache", "modules", modulehash)
+    else
+        buildir = path.join(projectdir, "cache", "modules", modulehash)
+    end
+    local envs = {XMAKE_CONFIGDIR = buildir}
+    local argv = {"config", "-o", buildir, "-a", xmake.arch()}
+    core_sandbox_module._add_builtin_argv(argv, projectdir)
+    os.execv(os.programfile(), argv, {envs = envs, curdir = projectdir})
+    argv = {}
+    core_sandbox_module._add_builtin_argv(argv, projectdir)
+    os.execv(os.programfile(), argv, {envs = envs, curdir = projectdir})
+    return buildir
+end
 
-                -- bind main entry
-                if type(result) == "table" and result.main then
-                    setmetatable(result, { __call = function (_, ...) return result.main(...) end})
+-- load module from the binary module
+function core_sandbox_module._load_from_binary(module_fullpath, opt)
+    local module
+    local module_buildir = core_sandbox_module._build_module(module_fullpath)
+    local binaryfiles = os.files(path.join(module_buildir, "module_*"))
+    if binaryfiles then
+        module = {}
+        for _, binaryfile in ipairs(binaryfiles) do
+            local modulename = path.basename(binaryfile):sub(8)
+            module[modulename] = function (...)
+                local argv = {}
+                for _, arg in ipairs(table.pack(...)) do
+                    table.insert(argv, tostring(arg))
                 end
-
-                -- get the module path
-                local modulepath = path.relative(modulefile, moduleroot)
-                if not modulepath then
-                    return nil, string.format("cannot get the path for module: %s", name)
+                local ok, outdata, errdata, errors = os.iorunv(binaryfile, argv)
+                if ok then
+                    return outdata
+                else
+                    if not errors then
+                        errors = errdata or ""
+                        if #errors:trim() == 0 then
+                            errors = outdata or ""
+                        end
+                    end
+                    os.raise({errors = errors, stderr = errdata, stdout = outdata})
                 end
+            end
+        end
+    end
+    return module
+end
 
-                -- init the root module
-                module = module or {}
-
-                -- save script
-                script = errors
-
-                -- save module
-                local scope = module
-                for _, modulename in ipairs(path.split(modulepath)) do
-
-                    -- is end?
-                    local pos = modulename:find(".lua", 1, true)
-                    if pos then
+-- load module from the shared module
+function core_sandbox_module._load_from_shared(module_fullpath, opt)
+    local script
+    local module
+    local module_buildir = core_sandbox_module._build_module(module_fullpath)
+    local libraryfiles = os.files(path.join(module_buildir, "*module_*"))
+    if libraryfiles then
+        for _, libraryfile in ipairs(libraryfiles) do
+            local modulename = path.basename(libraryfile):match("module_(.+)")
+            script, errors = package.loadlib(libraryfile, "luaopen_" .. modulename)
+            if not script then
+                return nil, errors
+            end
+            module = script()
+            if module then
+                break
+            end
+        end
+    end
+    return module, script
+end
 
-                        -- get the module name
-                        modulename = modulename:sub(1, pos - 1)
-                        assert(modulename)
+-- load module
+function core_sandbox_module._load(dir, name, opt)
+    opt = opt or {}
+    assert(dir and name)
 
-                        -- save module
-                        scope[modulename] = result
+    -- get module subpath
+    local module_subpath = core_sandbox_module._modulepath(name)
+    assert(module_subpath)
 
-                    -- is scope?
-                    else
+    -- get module full path
+    local module_fullpath = path.join(dir, module_subpath)
 
-                        -- enter submodule
-                        scope[modulename] = scope[modulename] or {}
-                        scope = scope[modulename]
-                    end
-                end
-            end
-        end
+    -- load module
+    local script
+    local module
+    local modulekind = opt.modulekind
+    if modulekind == MODULE_KIND_LUAFILE then
+        module, script = core_sandbox_module._load_from_scriptfile(module_fullpath, opt)
+    elseif modulekind == MODULE_KIND_LUADIR then
+        module, script = core_sandbox_module._load_from_scriptdir(module_fullpath, opt)
+    elseif modulekind == MODULE_KIND_BINARY then
+        module, script = core_sandbox_module._load_from_binary(module_fullpath, opt)
+    elseif modulekind == MODULE_KIND_SHARED then
+        module, script = core_sandbox_module._load_from_shared(module_fullpath, opt)
     end
-
-    -- this module not found?
     if not module then
-        return nil, string.format("module: %s not found!", name)
+        local errors = script
+        return nil, errors or string.format("module: %s not found!", name)
     end
-
-    -- return it
     return module, script
 end
 
 -- find and load module
-function core_sandbox_module._find_and_load(name, opt, instance, modules, modules_directories)
-
-    -- load module
+function core_sandbox_module._find_and_load(name, opt)
+    opt = opt or {}
     local found = false
     local errors = nil
     local module = nil
     local modulekey = nil
-    local isdirs = false
+    local modulekind = MODULE_KIND_LUAFILE
+    local modules = opt.modules
+    local modules_directories = opt.modules_directories
     local loadnext = false
     for idx, moduledir in ipairs(modules_directories) do
-
-        -- find module and key
-        modulekey, isdirs = core_sandbox_module._find(moduledir, name)
+        modulekey, modulekind = core_sandbox_module._find(moduledir, name)
         if modulekey then
-
-            -- load it from cache first
             local moduleinfo = modules[modulekey]
             if moduleinfo and not opt.nocache and not opt.inherit then
                 module = moduleinfo[1]
                 errors = moduleinfo[2]
             else
-
-                -- load it from the script file
-                module, errors = core_sandbox_module._load(   moduledir, name
-                                                            , idx < #modules_directories and instance or nil  -- last modules need not fork sandbox
-                                                            , module)
-
-
-                -- cache this module
+                module, errors = core_sandbox_module._load(moduledir, name, {
+                                                           instance = idx < #modules_directories and opt.instance or nil,  -- last modules need not fork sandbox
+                                                           module = module,
+                                                           modulekind = modulekind})
                 if not opt.nocache then
                     modules[modulekey] = {module, errors}
                 end
             end
-
-            -- continue to load?
-            if module and isdirs then
+            if module and modulekind == MODULE_KIND_LUADIR then
                 loadnext = true
             end
-
-            -- found
             found = true
-
-            -- end?
             if not loadnext then
                 break
             end
@@ -272,17 +348,10 @@ end
 
 -- get module name
 function core_sandbox_module.name(name)
-
-    -- check
-    assert(name)
-
-    -- find modulename
     local i = name:lastof(".", true)
     if i then
         name = name:sub(i + 1)
     end
-
-    -- get it
     return name
 end
 
@@ -313,8 +382,6 @@ end
 
 -- find module
 function core_sandbox_module.find(name)
-
-    -- find it from the module directories
     for _, moduledir in ipairs(core_sandbox_module.directories()) do
         if (core_sandbox_module._find(moduledir, name)) then
             return true
@@ -392,7 +459,11 @@ function core_sandbox_module.import(name, opt)
     local modules_directories = (opt.nolocal or not rootdir) and core_sandbox_module.directories() or table.join(rootdir, core_sandbox_module.directories())
 
     -- load module
-    local found, module, errors = core_sandbox_module._find_and_load(name, opt, instance, modules, modules_directories)
+    local loadopt = table.clone(opt) or {}
+    loadopt.instance = instance
+    loadopt.modules = modules
+    loadopt.modules_directories = modules_directories
+    local found, module, errors = core_sandbox_module._find_and_load(name, loadopt)
 
     -- not found? attempt to load module.interface
     if not found and not opt.inherit then
@@ -409,7 +480,7 @@ function core_sandbox_module.import(name, opt)
 
         -- load module.interface
         if module2_name and interface_name then
-            found2, module2, errors2 = core_sandbox_module._find_and_load(module2_name, opt, instance, modules, modules_directories)
+            found2, module2, errors2 = core_sandbox_module._find_and_load(module2_name, loadopt)
             if found2 and module2 and module2[interface_name] then
                 module = module2[interface_name]
                 found = true
diff --git a/xmake/core/sandbox/modules/xmake.lua b/xmake/core/sandbox/modules/xmake.lua
index d4e23984a..f4f36d23c 100644
--- a/xmake/core/sandbox/modules/xmake.lua
+++ b/xmake/core/sandbox/modules/xmake.lua
@@ -25,6 +25,7 @@ local xmake   = require("base/xmake")
 local sandbox_xmake = sandbox_xmake or {}
 
 -- inherit some builtin interfaces
+sandbox_xmake.arch        = xmake.arch
 sandbox_xmake.version     = xmake.version
 sandbox_xmake.branch      = xmake.branch
 sandbox_xmake.programdir  = xmake.programdir
diff --git a/xmake/repository/packages/g/git/xmake.lua b/xmake/repository/packages/g/git/xmake.lua
index b8418a005..a8314da8e 100644
--- a/xmake/repository/packages/g/git/xmake.lua
+++ b/xmake/repository/packages/g/git/xmake.lua
@@ -1,5 +1,4 @@
 package("git")
-
     set_kind("binary")
     set_homepage("https://git-scm.com/")
     set_description("A free and open source distributed version control system")
@@ -29,7 +28,7 @@ package("git")
         package:addenv("PATH", path.join("share", "MinGit", "cmd"))
     end)
 
-    on_install("macosx", "linux", function (package)
+    on_install("macosx", "linux", "bsd", function (package)
         import("package.manager.install_package")("git")
     end)
 
diff --git a/xmake/rules/module/xmake.lua b/xmake/rules/module/xmake.lua
new file mode 100644
index 000000000..f70c71d7e
--- /dev/null
+++ b/xmake/rules/module/xmake.lua
@@ -0,0 +1,39 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        xmake.lua
+--
+
+rule("module.binary")
+    on_load(function (target)
+        import("core.project.config")
+        target:set("kind", "binary")
+        target:set("basename", "module_" .. target:name())
+        target:set("targetdir", config.buildir())
+    end)
+
+rule("module.shared")
+    add_deps("utils.symbols.export_all")
+    on_load(function (target)
+        assert(not xmake.luajit(), "rule(module.shared) only support for lua54 runtime!")
+        import("core.project.config")
+        target:set("kind", "shared")
+        target:set("basename", "module_" .. target:name())
+        target:set("targetdir", config.buildir())
+        target:set("strip", "none")
+    end)
+