Merge pull request #6989 from xmake-io/async

Add async support in os APIs

26 files changed, 1280 insertions, 269 deletions

diff --git a/core/src/tbox/tbox b/core/src/tbox/tbox
-Subproject 9bb164af89169f9194be7663795d954b0578831
+Subproject c3a4be464e2e42efdff83ca4abf22134facf5bf
diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index 0e79140e6..61ba79af1 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -769,11 +769,25 @@ static tb_bool_t xm_engine_save_arguments(xm_engine_t* engine, tb_int_t argc, tb
     return tb_true;
 }
 
-static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn)
+static tb_bool_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** argv, tb_char_t* path, tb_size_t maxn)
 {
     // check
     tb_assert_and_check_return_val(engine && path && maxn, tb_false);
 
+    /* we cache it, because the current path will be changed in thread.
+     *
+     * The executable file compiled using cosmocc on macOS might retrieve a relative path to the programfile.
+     * If the root directory has been changed, the retrieved programfile will be a non-existent path.
+     */
+    static tb_char_t s_program_filepath[TB_PATH_MAXN] = {0};
+    if (s_program_filepath[0])
+    {
+        tb_strlcpy(path, s_program_filepath, maxn);
+        lua_pushstring(engine->lua, s_program_filepath);
+        lua_setglobal(engine->lua, "_PROGRAM_FILE");
+        return tb_true;
+    }
+
     tb_bool_t ok = tb_false;
     do
     {
@@ -813,7 +827,10 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg
         if (!_NSGetExecutablePath(path, &bufsize))
             ok = tb_true;
 #elif defined(XM_PROC_SELF_FILE)
-        // get the executale file path as program directory
+        /* get the executale file path as program directory
+         *
+         * @see it may be a relative path
+         */
         ssize_t size = readlink(XM_PROC_SELF_FILE, path, (size_t)maxn);
         if (size > 0 && size < maxn)
         {
@@ -878,6 +895,9 @@ static tb_size_t xm_engine_get_program_file(xm_engine_t* engine, tb_char_t** arg
         // trace
         tb_trace_d("programfile: %s", path);
 
+        // cache it
+        tb_strlcpy(s_program_filepath, path, sizeof(s_program_filepath));
+
         // save the directory to the global variable: _PROGRAM_FILE
         lua_pushstring(engine->lua, path);
         lua_setglobal(engine->lua, "_PROGRAM_FILE");
@@ -909,6 +929,15 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t*
     // check
     tb_assert_and_check_return_val(engine && path && maxn, tb_false);
 
+    static tb_char_t s_program_directory[TB_PATH_MAXN] = {0};
+    if (s_program_directory[0])
+    {
+        tb_strlcpy(path, s_program_directory, maxn);
+        lua_pushstring(engine->lua, s_program_directory);
+        lua_setglobal(engine->lua, "_PROGRAM_DIR");
+        return tb_true;
+    }
+
     tb_bool_t ok = tb_false;
     tb_char_t data[TB_PATH_MAXN] = {0};
     do
@@ -994,6 +1023,9 @@ static tb_bool_t xm_engine_get_program_directory(xm_engine_t* engine, tb_char_t*
         // trace
         tb_trace_d("programdir: %s", path);
 
+        // cache it
+        tb_strlcpy(s_program_directory, path, sizeof(s_program_directory));
+
         // save the directory to the global variable: _PROGRAM_DIR
         lua_pushstring(engine->lua, path);
         lua_setglobal(engine->lua, "_PROGRAM_DIR");
diff --git a/core/src/xmake/thread/event_wait.c b/core/src/xmake/thread/event_wait.c
index 0831036c6..d6d628311 100644
--- a/core/src/xmake/thread/event_wait.c
+++ b/core/src/xmake/thread/event_wait.c
@@ -15,7 +15,7 @@
  * Copyright (C) 2015-present, Xmake Open Source Community.
  *
  * @author      ruki
- * @file        thread_event_unlock.c
+ * @file        thread_event_wait.c
  *
  */
 
diff --git a/tests/modules/os/async_copy.lua b/tests/modules/os/async_copy.lua
new file mode 100644
index 000000000..751a940ae
--- /dev/null
+++ b/tests/modules/os/async_copy.lua
@@ -0,0 +1,34 @@
+local function _prepare_source(dir)
+    os.mkdir(dir)
+    io.writefile(path.join(dir, "foo.txt"), "foo")
+    io.writefile(path.join(dir, "bar.txt"), "bar")
+end
+
+function main()
+    local root = os.tmpfile() .. ".os_async_copy"
+    local srcdir = path.join(root, "src")
+    local dstdir = path.join(root, "dst")
+
+    _prepare_source(srcdir)
+    print("source prepared: %s", srcdir)
+
+    local files = os.files(path.join(srcdir, "*.txt"), {async = true})
+    assert(files and #files == 2)
+    print("async enumerate: %d files", #files)
+
+    os.cp(srcdir, dstdir, {async = true})
+    assert(os.isdir(dstdir))
+    print("async copy done: %s", dstdir)
+
+    files = os.files(path.join(dstdir, "*.txt"), {async = true})
+    assert(files and #files == 2)
+
+    os.rm(dstdir, {async = true})
+    assert(not os.isdir(dstdir))
+    print("dst removed async")
+
+    os.rm(srcdir, {async = true})
+    os.tryrm(root)
+    print("async copy test finished")
+end
+
diff --git a/tests/modules/os/async_scheduler.lua b/tests/modules/os/async_scheduler.lua
new file mode 100644
index 000000000..0a201f8f7
--- /dev/null
+++ b/tests/modules/os/async_scheduler.lua
@@ -0,0 +1,32 @@
+import("core.base.scheduler")
+
+local function _prepare_workspace(root)
+    os.mkdir(root)
+    for idx = 1, 4 do
+        io.writefile(path.join(root, string.format("file%d.txt", idx)), "xmake")
+    end
+end
+
+function main()
+    local root = os.tmpfile() .. ".os_async_sched"
+    _prepare_workspace(root)
+    print("workspace prepared: %s", root)
+
+    local group = "os_async_scheduler"
+    scheduler.co_group_begin(group, function ()
+        for idx = 1, 4 do
+            scheduler.co_start(function ()
+                local matches = os.files(path.join(root, string.format("file%d.txt", idx)), {async = true})
+                assert(matches and #matches == 1)
+                print("async match %d finished", idx)
+            end)
+        end
+
+        scheduler.co_start(function ()
+            print("async find all files in programdir...")
+            local files = os.files(path.join(os.programdir(), "**"), {async = true})
+            print("files: %d", #files)
+            print("async find all finished")
+        end)
+    end)
+end
diff --git a/tests/modules/os/test.lua b/tests/modules/os/test.lua
index 92890a234..1207cb615 100644
--- a/tests/modules/os/test.lua
+++ b/tests/modules/os/test.lua
@@ -149,3 +149,25 @@ function test_args(t)
     t:are_equal(os.args({'-DTEST="hello world"', '-DTEST2="hello world2"'}), '"-DTEST=\\\"hello world\\\"" "-DTEST2=\\\"hello world2\\\""')
 end
 
+function test_async(t)
+    local tmpdir = os.tmpfile() .. ".dir"
+    local tmpdir2 = os.tmpfile() .. ".dir"
+    io.writefile(path.join(tmpdir, "foo.txt"), "foo")
+    io.writefile(path.join(tmpdir, "bar.txt"), "bar")
+    local files = os.files(path.join(tmpdir, "*.txt"), {async = true})
+    t:require(files and #files == 2)
+
+    os.cp(tmpdir, tmpdir2, {async = true, detach = true})
+    t:require(not os.isdir(tmpdir2))
+
+    os.cp(tmpdir, tmpdir2, {async = true})
+    t:require(os.isdir(tmpdir2))
+
+    t:require(os.isdir(tmpdir))
+    os.rm(tmpdir, {async = true})
+    t:require(not os.isdir(tmpdir))
+
+    t:require(os.isdir(tmpdir2))
+    os.rm(tmpdir2, {async = true, detach = true})
+    t:require(os.isdir(tmpdir2))
+end
diff --git a/tests/runner.lua b/tests/runner.lua
index 5b8b0f5f5..5d154674d 100644
--- a/tests/runner.lua
+++ b/tests/runner.lua
@@ -18,7 +18,6 @@ function main(script, opt)
     local context = test_context(script)
 
     local root = path.directory(script)
-
     local verbose = option.get("verbose") or option.get("diagnosis")
 
     -- trace
@@ -26,7 +25,6 @@ function main(script, opt)
 
     -- get test functions
     local data = import("test", { rootdir = root, anonymous = true })
-
     if data.main then
         -- ignore everthing when we found a main function
         data = { test_main = data.main }
@@ -37,9 +35,12 @@ function main(script, opt)
 
     -- run test
     local succeed_count = 0
+    local start_time = os.mclock()
     for k, v in pairs(data) do
         if k:startswith("test") and type(v) == "function" then
-            if verbose then print(">>     running %s ...", k) end
+            if verbose then
+                print(">>     running %s ...", k)
+            end
             context.func = v
             context.funcname = k
             local result = try
@@ -69,7 +70,9 @@ function main(script, opt)
             succeed_count = succeed_count + 1
         end
     end
-    if verbose then print(">>   finished %d test method(s) ...", succeed_count) end
+    if verbose then
+        print(">>   finished %d test method(s), spent %0.02fs", succeed_count, (os.mclock() - start_time) / 1000)
+    end
 
     -- leave script directory
     os.cd(old_dir)
diff --git a/xmake/core/_xmake_main.lua b/xmake/core/_xmake_main.lua
index 00538c864..139e6d20c 100644
--- a/xmake/core/_xmake_main.lua
+++ b/xmake/core/_xmake_main.lua
@@ -40,6 +40,12 @@ xmake._EMBED            = _EMBED
 xmake._THREAD_CALLBACK  = _THREAD_CALLBACK
 xmake._THREAD_CALLINFO  = _THREAD_CALLINFO
 
+-- we need not any arguments in sub-thread.
+-- because it always uses CommandLineToArgvW() on windows, so we need to reset it.
+if _THREAD_CALLBACK then
+    xmake._ARGV = {}
+end
+
 -- In order to be compatible with updates from lower versions of engine core
 -- @see https://github.com/xmake-io/xmake/issues/1694#issuecomment-925507210
 if xmake._LUAJIT == nil then
diff --git a/xmake/core/base/option.lua b/xmake/core/base/option.lua
index d258a1451..6d46bddef 100644
--- a/xmake/core/base/option.lua
+++ b/xmake/core/base/option.lua
@@ -314,21 +314,12 @@ end
 function option.taskmenu(task)
     assert(option._MENU)
 
-    -- the current task
     task = task or option.taskname() or "main"
-
-    -- get the task menu
     local taskmenu = option._MENU[task]
     if type(taskmenu) == "function" then
-
-        -- load this task menu
         taskmenu = taskmenu()
-
-        -- save this task menu
         option._MENU[task] = taskmenu
     end
-
-    -- get it
     return taskmenu
 end
 
diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua
index 647c76c50..dcd5ab5d1 100644
--- a/xmake/core/base/os.lua
+++ b/xmake/core/base/os.lua
@@ -54,6 +54,16 @@ os.SYSERR_NOT_PERM    = 1
 os.SYSERR_NOT_FILEDIR = 2
 os.SYSERR_NOT_ACCESS  = 3
 
+-- get the async task
+function os._async_task()
+    local async_task = os._ASYNC_TASK
+    if async_task == nil then
+        async_task = require("base/private/async_task")
+        os._ASYNC_TASK = async_task
+    end
+    return async_task
+end
+
 -- copy single file or directory
 function os._cp(src, dst, rootdir, opt)
     opt = opt or {}
@@ -390,7 +400,15 @@ end
 --              end)
 -- @endcode
 --
-function os.match(pattern, mode, callback)
+function os.match(pattern, mode, opt)
+
+    -- do it in the asynchronous task
+    if type(opt) == "table" and opt.async and xmake.in_main_thread() then
+        return os._async_task().match(pattern, mode)
+    end
+
+    -- extract callback
+    local callback = type(opt) == "function" and opt or (type(opt) == "table" and opt.callback or nil)
 
     -- support path instance
     pattern = tostring(pattern)
@@ -483,18 +501,18 @@ end
 --
 -- @note only return {} without count to simplify code, e.g. table.unpack(os.dirs(""))
 --
-function os.dirs(pattern, callback)
-    return (os.match(pattern, 'd', callback))
+function os.dirs(pattern, opt)
+    return (os.match(pattern, 'd', opt))
 end
 
 -- match files
-function os.files(pattern, callback)
-    return (os.match(pattern, 'f', callback))
+function os.files(pattern, opt)
+    return (os.match(pattern, 'f', opt))
 end
 
 -- match files and directories
-function os.filedirs(pattern, callback)
-    return (os.match(pattern, 'a', callback))
+function os.filedirs(pattern, opt)
+    return (os.match(pattern, 'a', opt))
 end
 
 -- copy files or directories and we can reserve the source directory structure
@@ -511,6 +529,11 @@ function os.cp(srcpath, dstpath, opt)
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt and opt.async and xmake.in_main_thread() then
+        return os._async_task().cp(srcpath, dstpath, {detach = opt.detach})
+    end
+
     -- reserve the source directory structure if opt.rootdir is given
     local rootdir = opt and opt.rootdir
     if rootdir then
@@ -564,14 +587,19 @@ end
 
 -- remove files or directories
 function os.rm(filepath, opt)
+    opt = opt or {}
 
     -- check arguments
     if not filepath then
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt.async and xmake.in_main_thread() then
+       return os._async_task().rm(filepath, {detach = opt.detach})
+    end
+
     -- remove file or directories
-    opt = opt or {}
     filepath = tostring(filepath)
     local filepathes = os._match_wildcard_pathes(filepath)
     if type(filepathes) == "string" then
@@ -617,6 +645,12 @@ end
 function os.cd(dir)
     assert(dir)
 
+    -- we can only change directory in main thread
+    if not xmake.in_main_thread() then
+        local thread = require("base/thread")
+        os.raise("we cannot change directory in non-main thread(%s)", thread.running() or "unknown")
+    end
+
     -- support path instance
     dir = tostring(dir)
 
@@ -682,13 +716,18 @@ function os.mkdir(dir)
 end
 
 -- remove directories
-function os.rmdir(dir)
+function os.rmdir(dir, opt)
 
     -- check arguments
     if not dir then
         return false, string.format("invalid arguments!")
     end
 
+    -- do it in the asynchronous task
+    if opt and opt.async and xmake.in_main_thread() then
+        return os._async_task().rmdir(dir, {detach = opt.detach})
+    end
+
     -- support path instance
     dir = tostring(dir)
 
@@ -835,17 +874,10 @@ function os.runv(program, argv, opt)
             errors = string.format("cannot runv(%s), %s", cmd, errors and errors or "unknown reason")
         end
 
-        -- remove the temporary log file
         os.rm(logfile)
-
-        -- failed
         return false, errors
     end
-
-    -- remove the temporary log file
     os.rm(logfile)
-
-    -- ok
     return true
 end
 
diff --git a/xmake/core/base/private/async_task.lua b/xmake/core/base/private/async_task.lua
new file mode 100644
index 000000000..ca176816e
--- /dev/null
+++ b/xmake/core/base/private/async_task.lua
@@ -0,0 +1,466 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        async_task.lua
+--
+
+-- define module: async_task
+local async_task = async_task or {}
+
+-- load modules
+local os     = require("base/os")
+local utils  = require("base/utils")
+local thread = require("base/thread")
+local option = require("base/option")
+local path   = require("base/path")
+local pipe_event = require("base/private/pipe_event")
+
+-- the task status
+local is_stopped = false
+local is_started = false
+
+-- the task event and queue
+local task_event = nil
+local task_queue = nil
+local task_mutex = nil
+
+-- object pool for sharedata
+local sharedata_pool = {}
+
+function async_task._absolute_dirs(searchdirs)
+    local dirs = {}
+    if searchdirs then
+        for _, directory in ipairs(searchdirs) do
+            local dir = tostring(directory)
+            if #dir > 0 then
+                table.insert(dirs, path.absolute(dir))
+            end
+        end
+    end
+    return dirs
+end
+
+function async_task._get_event()
+    return pipe_event.new("async_task")
+end
+
+function async_task._put_event(event)
+    if event then
+        event:close()
+    end
+end
+
+-- get sharedata from pool or create new one
+function async_task._get_sharedata()
+    local sharedata = table.remove(sharedata_pool)
+    if not sharedata then
+        sharedata = thread.sharedata()
+    else
+        sharedata:clear()
+    end
+    return sharedata
+end
+
+-- return sharedata to pool
+function async_task._put_sharedata(sharedata)
+    if sharedata then
+        table.insert(sharedata_pool, sharedata)
+    end
+end
+
+-- the asynchronous task loop
+function async_task._loop(event, queue, mutex, is_stopped, is_diagnosis)
+    local os = require("base/os")
+    local try = require("sandbox/modules/try")
+    local thread = require("base/thread")
+
+    local function dprint(...)
+        if is_diagnosis then
+            print(...)
+        end
+    end
+
+    local function _restore_thread_objects(cmd)
+        if cmd.event_data then
+            local event, errors = thread._deserialize_object(cmd.event_data)
+            assert(event, errors or "failed to deserialize event")
+            cmd.event = event
+        end
+        if cmd.result_data then
+            local result, errors = thread._deserialize_object(cmd.result_data)
+            assert(result, errors or "failed to deserialize result")
+            cmd.result = result
+        end
+    end
+
+    local function _runcmd_cp(cmd)
+        os.cp(cmd.srcpath, cmd.dstpath)
+    end
+    local function _runcmd_rm(cmd)
+        os.rm(cmd.filepath)
+    end
+    local function _runcmd_rmdir(cmd)
+        os.rmdir(cmd.dir)
+    end
+    local function _runcmd_match(cmd)
+        return os.match(cmd.pattern, cmd.mode)
+    end
+    local function _runcmd_find_file(cmd)
+        local find_file = require("sandbox/modules/import/lib/detect/find_file")
+        return find_file._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {})
+    end
+    local function _runcmd_find_path(cmd)
+        local find_path = require("sandbox/modules/import/lib/detect/find_path")
+        return find_path._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {})
+    end
+    local function _runcmd_find_directory(cmd)
+        local find_directory = require("sandbox/modules/import/lib/detect/find_directory")
+        return find_directory._find_from_directories(cmd.name, cmd.searchdirs or {}, cmd.suffixes or {})
+    end
+    local function _runcmd_find_library(cmd)
+        local find_library = require("sandbox/modules/import/lib/detect/find_library")
+        return find_library._find_from_directories(cmd.names or {}, cmd.searchdirs or {}, cmd.kinds or {}, cmd.suffixes or {}, cmd.opt or {})
+    end
+    local runops = {
+        cp    = _runcmd_cp,
+        rm    = _runcmd_rm,
+        rmdir = _runcmd_rmdir,
+        match = _runcmd_match,
+        find_file = _runcmd_find_file,
+        find_path = _runcmd_find_path,
+        find_directory = _runcmd_find_directory,
+        find_library = _runcmd_find_library
+    }
+    local function _runcmd(cmd)
+
+        _restore_thread_objects(cmd)
+
+        local ok = true
+        local errors
+        local result_data
+        local runop = runops[cmd.kind]
+        if runop then
+            try
+            {
+                function ()
+                    local ret = runop(cmd)
+                    if ret then
+                        result_data = ret
+                    end
+                end,
+                catch
+                {
+                    function (errs)
+                        ok = false
+                        errors = tostring(errs)
+                    end
+                }
+            }
+        end
+
+        -- notify completion if event is provided
+        if cmd.event and cmd.result then
+            cmd.result:set({ok = ok, errors = errors, data = result_data})
+            cmd.event:post()
+        end
+    end
+
+    dprint("async_task: started")
+    while not is_stopped:get() do
+        if event:wait(-1) > 0 then
+
+            -- fetch all tasks from queue at once
+            local cmds = {}
+            mutex:lock()
+            while not queue:empty() do
+                local cmd = queue:pop()
+                if cmd then
+                    table.insert(cmds, cmd)
+                end
+            end
+            mutex:unlock()
+
+            -- execute tasks without holding lock
+            dprint("async_task: handling tasks(%d) ..", #cmds)
+            for _, cmd in ipairs(cmds) do
+                _runcmd(cmd)
+            end
+        end
+    end
+    dprint("async_task: exited")
+end
+
+-- start the asynchronous task
+function async_task._start()
+    assert(task_queue == nil and task_event == nil and task_mutex == nil)
+    task_event = thread.event()
+    task_queue = thread.queue()
+    task_mutex = thread.mutex()
+    local task_is_stopped = thread.sharedata()
+    local task_thread = thread.new(async_task._loop, {
+        name = "core.base.async_task", internal = true,
+        argv = {task_event, task_queue, task_mutex, task_is_stopped, option.get("diagnosis")}})
+    local ok, errors = task_thread:start()
+    if not ok then
+        return false, errors
+    end
+    os.atexit(function (errors)
+        if task_thread then
+            is_stopped = true
+
+            -- Perhaps the thread hasn't started yet.
+            -- Let's wait a while and let it finish executing the tasks in the current queue.
+            utils.dprint("async_task: wait the pending tasks(%d) for exiting ..", task_queue:size())
+            task_mutex:lock()
+            local is_empty = task_queue:empty()
+            task_mutex:unlock()
+            if not is_empty then
+                task_event:post()
+                os.sleep(300)
+            end
+            task_is_stopped:set(true)
+            task_event:post()
+
+            utils.dprint("async_task: wait thread for exiting ..")
+            task_thread:wait(-1)
+            utils.dprint("async_task: wait finished")
+        end
+    end)
+    return true
+end
+
+-- ensure the asynchronous task is started
+function async_task._ensure_started()
+    if is_stopped then
+        return false, string.format("async_task has been stopped")
+    end
+    if not is_started then
+        local ok, errors = async_task._start()
+        if ok then
+            is_started = true
+        end
+        return ok, errors
+    end
+    assert(task_queue and task_event)
+    return true
+end
+
+-- post task and wait for result
+function async_task._post_task(cmd, is_detach, return_data)
+    local cmd_event
+    local cmd_result
+
+    -- create pipe and result for non-detach mode
+    if not is_detach then
+        cmd_event = async_task._get_event()
+        if not cmd_event then
+            return false, "failed to acquire event"
+        end
+        cmd_result = async_task._get_sharedata()
+        cmd.result_data = thread._serialize_object(cmd_result)
+        if not cmd.result_data then
+            async_task._put_sharedata(cmd_result)
+            async_task._put_event(cmd_event)
+            return false, "failed to serialize sharedata"
+        end
+        cmd.event_data = thread._serialize_object(cmd_event)
+        if not cmd.event_data then
+            async_task._put_sharedata(cmd_result)
+            async_task._put_event(cmd_event)
+            return false, "failed to serialize event"
+        end
+    end
+
+    task_mutex:lock()
+    task_queue:push(cmd)
+    task_mutex:unlock()
+
+    task_event:post()
+
+    if is_detach then
+        return true
+    end
+
+    local wait_ok, wait_errors = cmd_event:wait(-1)
+
+    local result
+    if wait_ok then
+        result = cmd_result:get()
+    end
+    async_task._put_sharedata(cmd_result)
+    async_task._put_event(cmd_event)
+
+    if not wait_ok then
+        return false, wait_errors or "wait event failed"
+    end
+
+    if result and result.ok then
+        if return_data then
+            local data = result.data
+            if type(data) == "table" then
+                return data, #data
+            elseif data ~= nil then
+                return data
+            else
+                return nil, 0
+            end
+        else
+            return true
+        end
+    else
+        if return_data then
+            return nil, 0
+        else
+            return false, result and result.errors or "unknown error"
+        end
+    end
+end
+
+-- copy files or directories
+function async_task.cp(srcpath, dstpath, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return false, errors
+    end
+    local cmd = {kind = "cp", srcpath = path.absolute(tostring(srcpath)), dstpath = path.absolute(tostring(dstpath))}
+    return async_task._post_task(cmd, opt.detach, false)
+end
+
+-- remove files or directories
+function async_task.rm(filepath, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return false, errors
+    end
+    local cmd = {kind = "rm", filepath = path.absolute(tostring(filepath))}
+    return async_task._post_task(cmd, opt.detach, false)
+end
+
+-- remove directories
+function async_task.rmdir(dir, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return false, errors
+    end
+    local cmd = {kind = "rmdir", dir = path.absolute(tostring(dir))}
+    return async_task._post_task(cmd, opt.detach, false)
+end
+
+-- match files or directories
+function async_task.match(pattern, mode)
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return nil, errors
+    end
+    local cmd = {kind = "match", pattern = path.absolute(tostring(pattern)), mode = mode}
+    return async_task._post_task(cmd, false, true)
+end
+
+-- find file
+function async_task.find_file(name, searchdirs, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return nil, errors
+    end
+    local dirs = async_task._absolute_dirs(searchdirs)
+    local suffixes = {}
+    if opt.suffixes then
+        for _, suffix in ipairs(opt.suffixes) do
+            table.insert(suffixes, tostring(suffix))
+        end
+    end
+    local cmd = {kind = "find_file", name = tostring(name), searchdirs = dirs, suffixes = suffixes}
+    return async_task._post_task(cmd, false, true)
+end
+
+-- find path
+function async_task.find_path(name, searchdirs, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return nil, errors
+    end
+    local dirs = async_task._absolute_dirs(searchdirs)
+    local suffixes = {}
+    if opt.suffixes then
+        for _, suffix in ipairs(opt.suffixes) do
+            table.insert(suffixes, tostring(suffix))
+        end
+    end
+    local cmd = {kind = "find_path", name = tostring(name), searchdirs = dirs, suffixes = suffixes}
+    return async_task._post_task(cmd, false, true)
+end
+
+-- find directory
+function async_task.find_directory(name, searchdirs, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return nil, errors
+    end
+    local dirs = async_task._absolute_dirs(searchdirs)
+    local suffixes = {}
+    if opt.suffixes then
+        for _, suffix in ipairs(opt.suffixes) do
+            table.insert(suffixes, tostring(suffix))
+        end
+    end
+    local cmd = {kind = "find_directory", name = tostring(name), searchdirs = dirs, suffixes = suffixes}
+    return async_task._post_task(cmd, false, true)
+end
+
+-- find library
+function async_task.find_library(names, searchdirs, kinds, opt)
+    opt = opt or {}
+    local ok, errors = async_task._ensure_started()
+    if not ok then
+        return nil, errors
+    end
+    local dirs = async_task._absolute_dirs(searchdirs)
+    local suffixes = {}
+    if opt.suffixes then
+        for _, suffix in ipairs(opt.suffixes) do
+            table.insert(suffixes, tostring(suffix))
+        end
+    end
+    local names_list = {}
+    if names then
+        if type(names) == "table" then
+            for _, name in ipairs(names) do
+                table.insert(names_list, tostring(name))
+            end
+        else
+            table.insert(names_list, tostring(names))
+        end
+    end
+    local kinds_list = {}
+    if kinds then
+        for _, kind in ipairs(kinds) do
+            table.insert(kinds_list, tostring(kind))
+        end
+    end
+    local cmd = {kind = "find_library", names = names_list, searchdirs = dirs, kinds = kinds_list, suffixes = suffixes, opt = {plat = opt.plat}}
+    return async_task._post_task(cmd, false, true)
+end
+
+-- return module: async_task
+return async_task
+
diff --git a/xmake/core/base/private/pipe_event.lua b/xmake/core/base/private/pipe_event.lua
new file mode 100644
index 000000000..8d1278766
--- /dev/null
+++ b/xmake/core/base/private/pipe_event.lua
@@ -0,0 +1,140 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        pipe_event.lua
+--
+
+-- define module
+local pipe_event = pipe_event or {}
+local _instance = _instance or {}
+
+-- load modules
+local pipe   = require("base/pipe")
+local libc   = require("base/libc")
+local bytes  = require("base/bytes")
+local table  = require("base/table")
+
+function _instance.new(name)
+    local event = table.inherit(_instance)
+    event._PIPE_EVENT = true
+    event._BUFFER = bytes(2)
+    event._NAME = name or "pipe_event"
+    local reader, writer, errors = pipe.openpair("AA")
+    if not reader or not writer then
+        if reader then reader:close() end
+        if writer then writer:close() end
+        return nil, errors or "failed to open pipe"
+    end
+    event._READER = reader
+    event._WRITER = writer
+    event._WRITER_PTR = nil
+    return event
+end
+
+function _instance:name()
+    return self._NAME
+end
+
+function _instance:post()
+    local writer = self._WRITER
+    if not writer then
+        return false, "pipe event writer closed"
+    end
+    local ok, errors = writer:write("1", {block = true})
+    if ok < 0 then
+        return false, errors or "pipe event post failed"
+    end
+    writer:close()
+    self._WRITER = nil
+    self._WRITER_PTR = nil
+    return true
+end
+
+function _instance:wait(timeout)
+    if not self._READER then
+        return false, "pipe event reader closed"
+    end
+    local read, read_errors = self._READER:read(self._BUFFER, 1, {block = true, timeout = timeout})
+    if read < 0 then
+        return false, read_errors
+    end
+    return read
+end
+
+function _instance:close()
+    if self._READER then
+        self._READER:close()
+    end
+    if self._WRITER then
+        self._WRITER:close()
+    end
+    self._READER = nil
+    self._WRITER = nil
+    self._WRITER_PTR = nil
+end
+
+-- return pipe cdata for serialization (writer pointer is stable for passing across threads)
+function _instance:cdata()
+    if self._WRITER then
+        return self._WRITER:cdata()
+    end
+    return self._WRITER_PTR
+end
+
+function _instance:__gc()
+    self:close()
+end
+
+function _instance:_serialize()
+    if not self._WRITER and self._WRITER_PTR then
+        return {ptr = self._WRITER_PTR, name = self:name()}
+    end
+    if not self._WRITER then
+        return nil
+    end
+    local ptr = libc.dataptr(self._WRITER:cdata(), {ffi = false})
+    if not ptr then
+        return nil
+    end
+    self._WRITER._PIPE = nil
+    self._WRITER = nil
+    self._WRITER_PTR = ptr
+    return {ptr = ptr, name = self:name()}
+end
+
+function _instance:_deserialize(data)
+    if not data or not data.ptr then
+        return false, "invalid pipe event data"
+    end
+    self:close()
+    local writer = pipe.new(libc.ptraddr(data.ptr, {ffi = false}))
+    if not writer then
+        return false, "invalid pipe pointer"
+    end
+    self._NAME = data.name or self._NAME or "pipe_event"
+    self._WRITER = writer
+    self._WRITER_PTR = data.ptr
+    return true
+end
+
+function pipe_event.new(name)
+    return _instance.new(name)
+end
+
+return pipe_event
+
+
diff --git a/xmake/core/base/thread.lua b/xmake/core/base/thread.lua
index 3856feb81..26ac672c3 100644
--- a/xmake/core/base/thread.lua
+++ b/xmake/core/base/thread.lua
@@ -28,14 +28,15 @@ local _queue     = _queue or {}
 local _sharedata = _sharedata or {}
 
 -- load modules
-local io        = require("base/io")
-local libc      = require("base/libc")
-local pipe      = require("base/pipe")
-local bytes     = require("base/bytes")
-local table     = require("base/table")
-local string    = require("base/string")
-local scheduler = require("base/scheduler")
-local sandbox   = require("sandbox/sandbox")
+local io         = require("base/io")
+local libc       = require("base/libc")
+local pipe       = require("base/pipe")
+local pipe_event = require("base/private/pipe_event")
+local bytes      = require("base/bytes")
+local table      = require("base/table")
+local string     = require("base/string")
+local scheduler  = require("base/scheduler")
+local sandbox    = require("sandbox/sandbox")
 
 -- the thread status
 thread.STATUS_READY     = 1
@@ -52,6 +53,7 @@ function _thread.new(callback, opt)
     instance._CALLBACK  = callback
     instance._STACKSIZE = opt.stacksize or 0
     instance._STATUS    = thread.STATUS_READY
+    instance._INTERNAL  = opt.internal
     setmetatable(instance, _thread)
     return instance
 end
@@ -102,41 +104,28 @@ function _thread:start()
     local argv = {}
     for _, arg in ipairs(self._ARGV) do
         if type(arg) == "table" then
-            -- is mutex? we can only pass cdata address
-            if arg._MUTEX and arg.cdata then
-                thread.mutex_incref(arg:cdata())
-                arg = {mutex = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})}
-            -- is event? we can only pass cdata address
-            elseif arg._EVENT and arg.cdata then
-                thread.event_incref(arg:cdata())
-                arg = {event = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})}
-            -- is semaphore? we can only pass cdata address
-            elseif arg._SEMAPHORE and arg.cdata then
-                thread.semaphore_incref(arg:cdata())
-                arg = {semaphore = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})}
-            -- is queue? we can only pass cdata address
-            elseif arg._QUEUE and arg.cdata then
-                thread.queue_incref(arg:cdata())
-                arg = {queue = true, name = arg:name(), caddr = libc.dataptr(arg:cdata(), {ffi = false})}
-            -- is sharedata? we can only pass cdata address
-            elseif arg._SHAREDATA and arg.cdata then
-                thread.sharedata_incref(arg:cdata())
-                arg = {sharedata = true, name = arg:name(), caddr = libc.dataptr(arg:cdata())}
+            -- try to serialize thread object (mutex, event, semaphore, queue, sharedata)
+            local serialized = thread._serialize_object(arg)
+            if serialized then
+                arg = serialized
             end
         end
         table.insert(argv, arg)
     end
 
     -- init callback info
+    local is_internal = self._INTERNAL
     local callback = string._dump(self._CALLBACK)
-    local callinfo = {name = self:name(), argv = argv}
+    local callinfo = {name = self:name(), argv = argv, internal = is_internal}
 
     -- we need a pipe pair to wait and listen thread exit event
-    local rpipe, wpipe = pipe.openpair("AA")
-    self._RPIPE = rpipe
-    callinfo.wpipe = libc.dataptr(wpipe:cdata(), {ffi = false})
-    -- we need to suppress gc to free it, because it has been transfer to thread in another lua state instance
-    wpipe._PIPE = nil
+    if not is_internal then
+        local rpipe, wpipe = pipe.openpair("AA")
+        self._RPIPE = rpipe
+        callinfo.wpipe = libc.dataptr(wpipe:cdata(), {ffi = false})
+        -- we need to suppress gc to free it, because it has been transfer to thread in another lua state instance
+        wpipe._PIPE = nil
+    end
 
     -- serialize and pass callback and arguments to this thread
     -- we do not use string.serialize to serialize callback, because it's slower (deserialize)
@@ -206,8 +195,13 @@ function _thread:wait(timeout)
             ok = read
             errors = data_or_errors
         end
-    else
-        ok, errors = thread.thread_wait(self:cdata(), timeout)
+    end
+    if not rpipe then
+        local waitok, wait_errors = thread.thread_wait(self:cdata(), timeout)
+        if ok == nil or ok > 0 then
+            ok = waitok
+            errors = wait_errors
+        end
     end
     if ok < 0 then
         return -1, errors or string.format("%s: failed to resume thread!", self)
@@ -781,23 +775,90 @@ function _sharedata:__gc()
     end
 end
 
--- new a thread
---
--- @param callback      the thread callback
--- @param opt           the thread options, e.g. {name = "", argv = {}, stacksize = 8192}
---
--- @return the thread instance
---
-function thread.new(callback, opt)
-    if callback == nil then
-        return nil, "invalid thread, callback is nil"
+-- serialize thread object for passing through queue or table (private helper)
+-- this is used when you need to pass thread objects (mutex, event, semaphore, queue, sharedata)
+-- through a queue or embed them in a table
+-- returns a table with serialized caddr that can be pushed to queue
+function thread._serialize_object(obj)
+    if not obj or type(obj) ~= "table" or not obj.cdata then
+        return nil
     end
-    return _thread.new(callback, opt)
+
+    local result = {}
+    -- detect object type by checking internal marker
+    if obj._MUTEX then
+        thread.mutex_incref(obj:cdata())
+        result.mutex = true
+        result.name = obj:name()
+        result.caddr = libc.dataptr(obj:cdata(), {ffi = false})
+    elseif obj._EVENT then
+        thread.event_incref(obj:cdata())
+        result.event = true
+        result.name = obj:name()
+        result.caddr = libc.dataptr(obj:cdata(), {ffi = false})
+    elseif obj._SEMAPHORE then
+        thread.semaphore_incref(obj:cdata())
+        result.semaphore = true
+        result.name = obj:name()
+        result.caddr = libc.dataptr(obj:cdata(), {ffi = false})
+    elseif obj._QUEUE then
+        thread.queue_incref(obj:cdata())
+        result.queue = true
+        result.name = obj:name()
+        result.caddr = libc.dataptr(obj:cdata(), {ffi = false})
+    elseif obj._SHAREDATA then
+        thread.sharedata_incref(obj:cdata())
+        result.sharedata = true
+        result.name = obj:name()
+        result.caddr = libc.dataptr(obj:cdata(), {ffi = false})
+    elseif obj._PIPE_EVENT then
+        local data = obj:_serialize()
+        if not data then
+            return nil
+        end
+        result.pipe_event = true
+        result.ptr = data.ptr
+        result.name = data.name
+        result.caddr = data.ptr
+    else
+        return nil
+    end
+
+    return result
 end
 
--- get the running thread name
-function thread.running()
-    return thread._RUNNING
+-- deserialize thread object from serialized data (private helper)
+-- this is used to restore thread objects (mutex, event, semaphore, queue, sharedata)
+-- from serialized caddr received through queue or from table
+function thread._deserialize_object(data)
+    if not data or type(data) ~= "table" or not data.caddr then
+        return nil
+    end
+
+    local cdata = libc.ptraddr(data.caddr, {ffi = false})
+    if data.mutex then
+        return _mutex.new(data.name, cdata)
+    elseif data.event then
+        return _event.new(data.name, cdata)
+    elseif data.semaphore then
+        return _semaphore.new(data.name, cdata)
+    elseif data.queue then
+        return _queue.new(data.name, cdata)
+    elseif data.sharedata then
+        return _sharedata.new(data.name, cdata)
+    elseif data.pipe_event then
+        local event = pipe_event.new(data.name)
+        if not event then
+            return nil, "failed to create pipe event"
+        end
+        local ok, errors = event:_deserialize({ptr = data.ptr, name = data.name})
+        if not ok then
+            return nil, errors
+        end
+        return event
+    end
+
+    return nil
 end
 
 -- run thread
@@ -808,6 +869,7 @@ function thread._run_thread(callback_str, callinfo_str)
     local argv
     local threadname
     local wpipe
+    local is_internal = false
     if callinfo_str then
         local result, errors = string.deserialize(callinfo_str)
         if not result then
@@ -817,7 +879,10 @@ function thread._run_thread(callback_str, callinfo_str)
         if callinfo then
             argv = callinfo.argv
             threadname = callinfo.name
-            wpipe = pipe.new(libc.ptraddr(callinfo.wpipe, {ffi = false}))
+            is_internal = callinfo.internal
+            if callinfo.wpipe then
+                wpipe = pipe.new(libc.ptraddr(callinfo.wpipe, {ffi = false}))
+            end
         end
     end
 
@@ -850,23 +915,24 @@ function thread._run_thread(callback_str, callinfo_str)
         return false, errors
     end
 
+    -- if it's an internal thread, we need to bind some additional internal interfaces.
+    if is_internal then
+        sandbox_inst:api_register_builtin("require", require)
+    end
+
     -- save the running thread name
     thread._RUNNING = threadname
 
-    -- translate arguments (mutex, ...)
+    -- translate arguments (mutex, event, semaphore, queue, sharedata, ...)
     if argv then
         local newargv = {}
         for _, arg in ipairs(argv) do
-            if type(arg) == "table" and arg.mutex and arg.caddr then
-                arg = _mutex.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false}))
-            elseif type(arg) == "table" and arg.event and arg.caddr then
-                arg = _event.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false}))
-            elseif type(arg) == "table" and arg.semaphore and arg.caddr then
-                arg = _semaphore.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false}))
-            elseif type(arg) == "table" and arg.queue and arg.caddr then
-                arg = _queue.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false}))
-            elseif type(arg) == "table" and arg.sharedata and arg.caddr then
-                arg = _sharedata.new(arg.name, libc.ptraddr(arg.caddr, {ffi = false}))
+            if type(arg) == "table" and arg.caddr then
+                -- try to deserialize thread object
+                local obj = thread._deserialize_object(arg)
+                if obj then
+                    arg = obj
+                end
             end
             table.insert(newargv, arg)
         end
@@ -887,6 +953,25 @@ function thread._run_thread(callback_str, callinfo_str)
     return ok, errors
 end
 
+-- new a thread
+--
+-- @param callback      the thread callback
+-- @param opt           the thread options, e.g. {name = "", argv = {}, stacksize = 8192}
+--
+-- @return the thread instance
+--
+function thread.new(callback, opt)
+    if callback == nil then
+        return nil, "invalid thread, callback is nil"
+    end
+    return _thread.new(callback, opt)
+end
+
+-- get the running thread name
+function thread.running()
+    return thread._RUNNING
+end
+
 -- open a mutex
 function thread.mutex(name)
     local mutex = thread.mutex_init()
diff --git a/xmake/core/base/utils.lua b/xmake/core/base/utils.lua
index d20b3c774..ac9884851 100644
--- a/xmake/core/base/utils.lua
+++ b/xmake/core/base/utils.lua
@@ -193,6 +193,20 @@ function utils.vprintf(format, ...)
     end
 end
 
+-- print the diagnosis information
+function utils.dprint(format, ...)
+    if option.get("diagnosis") and format ~= nil then
+        utils.print(format, ...)
+    end
+end
+
+-- print the diagnosis information without newline
+function utils.dprintf(format, ...)
+    if option.get("diagnosis") and format ~= nil then
+        utils.printf(format, ...)
+    end
+end
+
 -- print the error information
 function utils.error(format, ...)
     if format ~= nil then
diff --git a/xmake/core/base/xmake.lua b/xmake/core/base/xmake.lua
index 2b41e9a9d..d25e60147 100644
--- a/xmake/core/base/xmake.lua
+++ b/xmake/core/base/xmake.lua
@@ -67,6 +67,11 @@ function xmake.is_embed()
     return xmake._EMBED or false
 end
 
+-- in main thread?
+function xmake.in_main_thread()
+    return xmake._THREAD_CALLBACK == nil
+end
+
 -- get command arguments
 function xmake.argv()
     return xmake._ARGV
diff --git a/xmake/core/main.lua b/xmake/core/main.lua
index 3f0a4484f..1c51342b9 100644
--- a/xmake/core/main.lua
+++ b/xmake/core/main.lua
@@ -52,9 +52,11 @@ local menu =
     -- the tasks: xmake [task]
 ,   function ()
         local tasks = task.tasks() or {}
-        local ok, project_tasks = pcall(project.tasks)
-        if ok then
-            table.join2(tasks, project_tasks)
+        if xmake.in_main_thread() then
+            local ok, project_tasks = pcall(project.tasks)
+            if ok then
+                table.join2(tasks, project_tasks)
+            end
         end
         return task.menu(tasks)
     end
@@ -218,7 +220,7 @@ function main._init()
         xmake._PROJECT_FILE = projectfile
 
         -- enter the project directory
-        if os.isdir(os.projectdir()) then
+        if os.isdir(os.projectdir()) and xmake.in_main_thread() then
             os.cd(os.projectdir())
         end
     else
@@ -302,33 +304,36 @@ function main.entry()
         return main._exit(ok, errors)
     end
 
-    -- check run command as root
-    if main._limit_root() then
-        if os.isroot() then
-            errors = [[Running xmake as root is extremely dangerous and no longer supported.
-As xmake does not drop privileges on installation you would be giving all
-build scripts full access to your system.
-Or you can add `--root` option or XMAKE_ROOT=y to allow run as root temporarily.
-            ]]
-            return main._exit(false, errors)
+    if xmake.in_main_thread() then
+
+        -- check run command as root
+        if main._limit_root() then
+            if os.isroot() then
+                errors = [[Running xmake as root is extremely dangerous and no longer supported.
+    As xmake does not drop privileges on installation you would be giving all
+    build scripts full access to your system.
+    Or you can add `--root` option or XMAKE_ROOT=y to allow run as root temporarily.
+                ]]
+                return main._exit(false, errors)
+            end
         end
-    end
 
-    -- show help?
-    if main._show_help() then
-        return main._exit(true)
-    end
+        -- show help?
+        if main._show_help() then
+            return main._exit(true)
+        end
 
-    -- save command lines to history and we need to make sure that the .xmake directory is not generated everywhere
-    local skip_history = (os.getenv('XMAKE_SKIP_HISTORY') or ''):trim()
-    if os.projectfile() and os.isfile(os.projectfile()) and os.isdir(config.directory()) and skip_history == '' then
-        local cmdlines = table.wrap(localcache.get("history", "cmdlines"))
-        if #cmdlines > 64 then
-            table.remove(cmdlines, 1)
+        -- save command lines to history and we need to make sure that the .xmake directory is not generated everywhere
+        local skip_history = (os.getenv('XMAKE_SKIP_HISTORY') or ''):trim()
+        if os.projectfile() and os.isfile(os.projectfile()) and os.isdir(config.directory()) and skip_history == '' then
+            local cmdlines = table.wrap(localcache.get("history", "cmdlines"))
+            if #cmdlines > 64 then
+                table.remove(cmdlines, 1)
+            end
+            table.insert(cmdlines, option.cmdline())
+            localcache.set("history", "cmdlines", cmdlines)
+            localcache.save("history")
         end
-        table.insert(cmdlines, option.cmdline())
-        localcache.set("history", "cmdlines", cmdlines)
-        localcache.save("history")
     end
 
     -- enable scheduler
@@ -362,3 +367,4 @@ end
 
 -- return module: main
 return main
+
diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua
index 8b94cbff7..dc54507a9 100644
--- a/xmake/core/project/project.lua
+++ b/xmake/core/project/project.lua
@@ -1276,6 +1276,11 @@ end
 -- get the project menu
 function project.menu()
 
+    -- we cannot only get it in main thread
+    if not xmake.in_main_thread() then
+        return {}
+    end
+
     -- attempt to load options from the project file
     local options = nil
     local errors = nil
diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua b/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua
index 4c6048428..ec737785b 100644
--- a/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua
+++ b/xmake/core/sandbox/modules/import/lib/detect/find_directory.lua
@@ -28,6 +28,66 @@ local utils     = require("base/utils")
 local table     = require("base/table")
 local raise     = require("sandbox/modules/raise")
 local vformat   = require("sandbox/modules/vformat")
+local xmake     = require("base/xmake")
+
+-- expand search paths
+function sandbox_lib_detect_find_directory._expand_paths(paths)
+    local results = {}
+    for _, _path in ipairs(table.wrap(paths)) do
+        if type(_path) == "function" then
+            local ok, result_or_errors = sandbox.load(_path)
+            if ok then
+                _path = result_or_errors or ""
+            else
+                raise(result_or_errors)
+            end
+        else
+            _path = vformat(_path)
+        end
+        for _, _s_path in ipairs(table.wrap(_path)) do
+            _s_path = tostring(_s_path)
+            if #_s_path > 0 then
+                table.insert(results, _s_path)
+            end
+        end
+    end
+    return results
+end
+
+-- normalize suffixes
+function sandbox_lib_detect_find_directory._normalize_suffixes(suffixes)
+    local results = {}
+    for _, suffix in ipairs(table.wrap(suffixes)) do
+        suffix = tostring(suffix)
+        if #suffix > 0 then
+            table.insert(results, suffix)
+        end
+    end
+    return results
+end
+
+-- find directory from directories list
+function sandbox_lib_detect_find_directory._find_from_directories(name, directories, suffixes)
+    directories = table.wrap(directories)
+    suffixes = table.wrap(suffixes)
+    if #suffixes > 0 then
+        for _, directory in ipairs(directories) do
+            for _, suffix in ipairs(suffixes) do
+                local results = os.dirs(path.join(directory, suffix, name), function (file, isdir) return false end)
+                if results and #results > 0 then
+                    return results[1]
+                end
+            end
+        end
+    else
+        for _, directory in ipairs(directories) do
+            local results = os.dirs(path.join(directory, name), function (file, isdir) return false end)
+            if results and #results > 0 then
+                return results[1]
+            end
+        end
+    end
+end
 
 -- find directory
 --
@@ -51,43 +111,17 @@ function sandbox_lib_detect_find_directory.main(name, paths, opt)
     -- init options
     opt = opt or {}
 
-    -- init paths
-    paths = table.wrap(paths)
+    local suffixes = sandbox_lib_detect_find_directory._normalize_suffixes(opt.suffixes)
+    local directories = sandbox_lib_detect_find_directory._expand_paths(paths)
 
-    -- append suffixes to paths
-    local suffixes = table.wrap(opt.suffixes)
-    if #suffixes > 0 then
-        local paths_new = {}
-        for _, parent in ipairs(paths) do
-            for _, suffix in ipairs(suffixes) do
-                table.insert(paths_new, path.join(parent, suffix))
-            end
-        end
-        paths = paths_new
+    if opt.async and xmake.in_main_thread() then
+        local result, _ = os._async_task().find_directory(name, directories, {suffixes = suffixes})
+        return result
     end
 
-    -- find file
-    for _, _path in ipairs(paths) do
-
-        -- format path for builtin variables
-        if type(_path) == "function" then
-            local ok, results = sandbox.load(_path)
-            if ok then
-                _path = results or ""
-            else
-                raise(results)
-            end
-        else
-            _path = vformat(_path)
-        end
-
-        -- find the first directory
-        local results = os.dirs(path.join(_path, name), function (file, isdir) return false end)
-        if results and #results > 0 then
-            return results[1]
-        end
-    end
+    return sandbox_lib_detect_find_directory._find_from_directories(name, directories, suffixes)
 end
 
 -- return module
 return sandbox_lib_detect_find_directory
+
diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_file.lua b/xmake/core/sandbox/modules/import/lib/detect/find_file.lua
index 8c82d5546..71e580ba1 100644
--- a/xmake/core/sandbox/modules/import/lib/detect/find_file.lua
+++ b/xmake/core/sandbox/modules/import/lib/detect/find_file.lua
@@ -29,6 +29,72 @@ local table     = require("base/table")
 local profiler  = require("base/profiler")
 local raise     = require("sandbox/modules/raise")
 local vformat   = require("sandbox/modules/vformat")
+local xmake     = require("base/xmake")
+
+-- expand search paths
+function sandbox_lib_detect_find_file._expand_paths(paths)
+    local results = {}
+    for _, _path in ipairs(table.wrap(paths)) do
+        if type(_path) == "function" then
+            local ok, result_or_errors = sandbox.load(_path)
+            if ok then
+                _path = result_or_errors or ""
+            else
+                raise(result_or_errors)
+            end
+        elseif type(_path) == "string" then
+            if _path:match("^%$%(env .+%)$") then
+                _path = path.splitenv(vformat(_path))
+            else
+                _path = vformat(_path)
+            end
+        end
+        for _, _s_path in ipairs(table.wrap(_path)) do
+            _s_path = tostring(_s_path)
+            if #_s_path > 0 then
+                table.insert(results, _s_path)
+            end
+        end
+    end
+    return results
+end
+
+-- normalize suffixes
+function sandbox_lib_detect_find_file._normalize_suffixes(suffixes)
+    local results = {}
+    for _, suffix in ipairs(table.wrap(suffixes)) do
+        suffix = tostring(suffix)
+        if #suffix > 0 then
+            table.insert(results, suffix)
+        end
+    end
+    return results
+end
+
+-- find from directories list
+function sandbox_lib_detect_find_file._find_from_directories(name, directories, suffixes)
+    local results
+    suffixes = table.wrap(suffixes)
+    directories = table.wrap(directories)
+    if #suffixes > 0 then
+        for _, directory in ipairs(directories) do
+            for _, suffix in ipairs(suffixes) do
+                local filedir = path.join(directory, suffix)
+                results = sandbox_lib_detect_find_file._find(filedir, name)
+                if results then
+                    return results
+                end
+            end
+        end
+    else
+        for _, directory in ipairs(directories) do
+            results = sandbox_lib_detect_find_file._find(directory, name)
+            if results then
+                return results
+            end
+        end
+    end
+end
 
 -- find the given file path or directory
 function sandbox_lib_detect_find_file._find(filedir, name)
@@ -72,48 +138,17 @@ function sandbox_lib_detect_find_file.main(name, paths, opt)
 
     -- find file
     profiler:enter("find_file", name)
-    local results
-    local suffixes = table.wrap(opt.suffixes)
-    for _, _path in ipairs(table.wrap(paths)) do
-
-        -- format path for builtin variables
-        if type(_path) == "function" then
-            local ok, result_or_errors = sandbox.load(_path)
-            if ok then
-                _path = result_or_errors or ""
-            else
-                raise(result_or_errors)
-            end
-        elseif type(_path) == "string" then
-            if _path:match("^%$%(env .+%)$") then
-                _path = path.splitenv(vformat(_path))
-            else
-                _path = vformat(_path)
-            end
-        end
+    local suffixes = sandbox_lib_detect_find_file._normalize_suffixes(opt.suffixes)
+    local directories = sandbox_lib_detect_find_file._expand_paths(paths)
 
-        for _, _s_path in ipairs(table.wrap(_path)) do
-            if #_s_path > 0 then
-                -- find file with suffixes
-                if #suffixes > 0 then
-                    for _, suffix in ipairs(suffixes) do
-                        local filedir = path.join(_s_path, suffix)
-                        results = sandbox_lib_detect_find_file._find(filedir, name)
-                        if results then
-                            goto found
-                        end
-                    end
-                else
-                    -- find file in the given path
-                    results = sandbox_lib_detect_find_file._find(_s_path, name)
-                    if results then
-                        goto found
-                    end
-                end
-            end
-        end
+    -- run in async task?
+    if opt.async and xmake.in_main_thread() then
+        local result, _ = os._async_task().find_file(name, directories, {suffixes = suffixes})
+        profiler:leave("find_file", name)
+        return result
     end
-::found::
+
+    local results = sandbox_lib_detect_find_file._find_from_directories(name, directories, suffixes)
     profiler:leave("find_file", name)
     return results
 end
diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_library.lua b/xmake/core/sandbox/modules/import/lib/detect/find_library.lua
index e072dded1..ab81e88b8 100644
--- a/xmake/core/sandbox/modules/import/lib/detect/find_library.lua
+++ b/xmake/core/sandbox/modules/import/lib/detect/find_library.lua
@@ -30,8 +30,40 @@ local config            = require("project/config")
 local target            = require("project/target")
 local raise             = require("sandbox/modules/raise")
 local import            = require("sandbox/modules/import")
+local xmake             = require("base/xmake")
 local find_file         = import("lib.detect.find_file")
 
+-- find library from directories list
+function sandbox_lib_detect_find_library._find_from_directories(names, directories, kinds, suffixes, opt)
+    names = table.wrap(names)
+    directories = table.wrap(directories)
+    kinds = table.wrap(kinds)
+    suffixes = table.wrap(suffixes)
+    if #kinds == 0 then
+        kinds = {"static", "shared"}
+    end
+    opt = opt or {}
+    local plat = opt.plat
+    for _, name in ipairs(names) do
+        for _, kind in ipairs(kinds) do
+            local filename = target.filename(name, kind, {plat = plat})
+            local filepath = find_file._find_from_directories(filename, directories, suffixes)
+            if plat == "mingw" then
+                if not filepath and kind == "shared" then
+                    filepath = find_file._find_from_directories(filename .. ".a", directories, suffixes)
+                end
+                if not filepath then
+                    filepath = find_file._find_from_directories(target.filename(name, kind, {plat = "windows"}), directories, suffixes)
+                end
+            end
+            if filepath then
+                local linkname = target.linkname(path.filename(filepath), {plat = plat})
+                return {kind = kind, filename = path.filename(filepath), linkdir = path.directory(filepath), link = linkname}
+            end
+        end
+    end
+end
+
 -- find library
 --
 -- @param names     the library names
@@ -56,27 +88,16 @@ function sandbox_lib_detect_find_library.main(names, paths, opt)
 
     -- find library file from the given paths
     opt = opt or {}
-    local kinds = opt.kind or {"static", "shared"}
-    for _, name in ipairs(table.wrap(names)) do
-        for _, kind in ipairs(table.wrap(kinds)) do
-            local filepath = find_file(target.filename(name, kind, {plat = opt.plat}), paths, opt)
-            if opt.plat == "mingw" then
-                if not filepath and kind == "shared" then
-                    -- for implib/mingw, e.g. libxxx.dll.a
-                    filepath = find_file(target.filename(name, kind, {plat = opt.plat}) .. ".a", paths, opt)
-                end
-                if not filepath then
-                    -- in order to be compatible with mingw/windows library with .lib
-                    filepath = find_file(target.filename(name, kind, {plat = "windows"}), paths, opt)
-                end
-            end
-            if filepath then
-                local filename = path.filename(filepath)
-                local linkname = target.linkname(filename, {plat = opt.plat})
-                return {kind = kind, filename = filename, linkdir = path.directory(filepath), link = linkname}
-            end
-        end
+    local directories = find_file._expand_paths(paths)
+    local suffixes = find_file._normalize_suffixes(opt.suffixes)
+    local kinds = table.wrap(opt.kind or {"static", "shared"})
+
+    if opt.async and xmake.in_main_thread() then
+        local result, _ = os._async_task().find_library(names, directories, kinds, {suffixes = suffixes, plat = opt.plat})
+        return result
     end
+
+    return sandbox_lib_detect_find_library._find_from_directories(names, directories, kinds, suffixes, {plat = opt.plat})
 end
 
 -- return module
diff --git a/xmake/core/sandbox/modules/import/lib/detect/find_path.lua b/xmake/core/sandbox/modules/import/lib/detect/find_path.lua
index f6a9ec426..cd7413fb7 100644
--- a/xmake/core/sandbox/modules/import/lib/detect/find_path.lua
+++ b/xmake/core/sandbox/modules/import/lib/detect/find_path.lua
@@ -28,6 +28,43 @@ local table     = require("base/table")
 local profiler  = require("base/profiler")
 local raise     = require("sandbox/modules/raise")
 local vformat   = require("sandbox/modules/vformat")
+local xmake     = require("base/xmake")
+
+-- expand search paths
+function sandbox_lib_detect_find_path._expand_paths(paths)
+    local results = {}
+    for _, _path in ipairs(table.wrap(paths)) do
+        if type(_path) == "function" then
+            local ok, result_or_errors = sandbox.load(_path)
+            if ok then
+                _path = result_or_errors or ""
+            else
+                raise(result_or_errors)
+            end
+        else
+            _path = vformat(_path)
+        end
+        for _, _s_path in ipairs(table.wrap(_path)) do
+            _s_path = tostring(_s_path)
+            if #_s_path > 0 then
+                table.insert(results, _s_path)
+            end
+        end
+    end
+    return results
+end
+
+-- normalize suffixes
+function sandbox_lib_detect_find_path._normalize_suffixes(suffixes)
+    local results = {}
+    for _, suffix in ipairs(table.wrap(suffixes)) do
+        suffix = tostring(suffix)
+        if #suffix > 0 then
+            table.insert(results, suffix)
+        end
+    end
+    return results
+end
 
 -- find the given file path or directory
 function sandbox_lib_detect_find_path._find(filedir, name)
@@ -52,6 +89,31 @@ function sandbox_lib_detect_find_path._find(filedir, name)
     end
 end
 
+-- find from directories list
+function sandbox_lib_detect_find_path._find_from_directories(name, directories, suffixes)
+    local results
+    suffixes = table.wrap(suffixes)
+    directories = table.wrap(directories)
+    if #suffixes > 0 then
+        for _, directory in ipairs(directories) do
+            for _, suffix in ipairs(suffixes) do
+                local filedir = path.join(directory, suffix)
+                results = sandbox_lib_detect_find_path._find(filedir, name)
+                if results then
+                    return results
+                end
+            end
+        end
+    else
+        for _, directory in ipairs(directories) do
+            results = sandbox_lib_detect_find_path._find(directory, name)
+            if results then
+                return results
+            end
+        end
+    end
+end
+
 -- find path
 --
 -- @param name      the path name
@@ -76,42 +138,17 @@ function sandbox_lib_detect_find_path.main(name, paths, opt)
     -- init options
     opt = opt or {}
 
-    -- find path
-    local results
     profiler:enter("find_path", name)
-    local suffixes = table.wrap(opt.suffixes)
-    for _, _path in ipairs(table.wrap(paths)) do
+    local suffixes = sandbox_lib_detect_find_path._normalize_suffixes(opt.suffixes)
+    local directories = sandbox_lib_detect_find_path._expand_paths(paths)
 
-        -- format path for builtin variables
-        if type(_path) == "function" then
-            local ok, result_or_errors = sandbox.load(_path)
-            if ok then
-                _path = result_or_errors or ""
-            else
-                raise(result_or_errors)
-            end
-        else
-            _path = vformat(_path)
-        end
-
-        -- find file with suffixes
-        if #suffixes > 0 then
-            for _, suffix in ipairs(suffixes) do
-                local filedir = path.join(_path, suffix)
-                results = sandbox_lib_detect_find_path._find(filedir, name)
-                if results then
-                    goto found
-                end
-            end
-        else
-            -- find file in the given path
-            results = sandbox_lib_detect_find_path._find(_path, name)
-            if results then
-                goto found
-            end
-        end
+    if opt.async and xmake.in_main_thread() then
+        local result, _ = os._async_task().find_path(name, directories, {suffixes = suffixes})
+        profiler:leave("find_path", name)
+        return result
     end
-::found::
+
+    local results = sandbox_lib_detect_find_path._find_from_directories(name, directories, suffixes)
     profiler:leave("find_path", name)
     return results
 end
diff --git a/xmake/core/sandbox/modules/os.lua b/xmake/core/sandbox/modules/os.lua
index 580e9e31d..aebb3cfb0 100644
--- a/xmake/core/sandbox/modules/os.lua
+++ b/xmake/core/sandbox/modules/os.lua
@@ -225,9 +225,9 @@ function sandbox_os.mkdir(dir)
 end
 
 -- remove directories
-function sandbox_os.rmdir(dir)
+function sandbox_os.rmdir(dir, opt)
     assert(dir)
-    local ok, errors = os.rmdir(vformat(dir))
+    local ok, errors = os.rmdir(vformat(dir), opt)
     if not ok then
         os.raise(errors)
     end
@@ -411,23 +411,23 @@ function sandbox_os.vexecv(program, argv, opt)
 end
 
 -- match files or directories
-function sandbox_os.match(pattern, mode, callback)
-    return os.match(vformat(tostring(pattern)), mode, callback)
+function sandbox_os.match(pattern, mode, opt)
+    return os.match(vformat(tostring(pattern)), mode, opt)
 end
 
 -- match directories
-function sandbox_os.dirs(pattern, callback)
-    return (sandbox_os.match(pattern, 'd', callback))
+function sandbox_os.dirs(pattern, opt)
+    return os.dirs(vformat(tostring(pattern)), opt)
 end
 
 -- match files
-function sandbox_os.files(pattern, callback)
-    return (sandbox_os.match(pattern, 'f', callback))
+function sandbox_os.files(pattern, opt)
+    return os.files(vformat(tostring(pattern)), opt)
 end
 
 -- match files and directories
-function sandbox_os.filedirs(pattern, callback)
-    return (sandbox_os.match(pattern, 'a', callback))
+function sandbox_os.filedirs(pattern, opt)
+    return os.filedirs(vformat(tostring(pattern)), opt)
 end
 
 -- is directory?
diff --git a/xmake/core/sandbox/modules/xmake.lua b/xmake/core/sandbox/modules/xmake.lua
index 8c040a9a0..b83536f4a 100644
--- a/xmake/core/sandbox/modules/xmake.lua
+++ b/xmake/core/sandbox/modules/xmake.lua
@@ -25,14 +25,15 @@ local xmake   = require("base/xmake")
 local sandbox_xmake = sandbox_xmake or {}
 
 -- inherit some builtin interfaces
-sandbox_xmake.arch        = xmake.arch
-sandbox_xmake.version     = xmake.version
-sandbox_xmake.branch      = xmake.branch
-sandbox_xmake.programdir  = xmake.programdir
-sandbox_xmake.programfile = xmake.programfile
-sandbox_xmake.luajit      = xmake.luajit
-sandbox_xmake.is_embed    = xmake.is_embed
-sandbox_xmake.argv        = xmake.argv
+sandbox_xmake.arch           = xmake.arch
+sandbox_xmake.version        = xmake.version
+sandbox_xmake.branch         = xmake.branch
+sandbox_xmake.programdir     = xmake.programdir
+sandbox_xmake.programfile    = xmake.programfile
+sandbox_xmake.luajit         = xmake.luajit
+sandbox_xmake.is_embed       = xmake.is_embed
+sandbox_xmake.in_main_thread = xmake.in_main_thread
+sandbox_xmake.argv           = xmake.argv
 
 -- return module
 return sandbox_xmake
diff --git a/xmake/core/sandbox/sandbox.lua b/xmake/core/sandbox/sandbox.lua
index c3fa92a2f..758a48030 100644
--- a/xmake/core/sandbox/sandbox.lua
+++ b/xmake/core/sandbox/sandbox.lua
@@ -245,6 +245,11 @@ function sandbox:namespace()
     return self._PRIVATE._NAMESPACE
 end
 
+-- register api for builtin
+function sandbox:api_register_builtin(name, func)
+    sandbox._api_register_builtin(self, name, func)
+end
+
 -- get current instance in the sandbox modules
 function sandbox.instance(script)
 
diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua
index d3c0094d8..2e7e6be89 100644
--- a/xmake/modules/private/cache/build_cache.lua
+++ b/xmake/modules/private/cache/build_cache.lua
@@ -339,7 +339,7 @@ function build(program, argv, opt)
                 _g.cache_miss_total_time = (_g.cache_miss_total_time or 0) + (os.mclock() - cache_miss_start_time)
             end
         end
-        os.rm(cppinfo.cppfile)
+        os.tryrm(cppinfo.cppfile)
     else
         _g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1
     end
diff --git a/xmake/modules/utils/run_script.lua b/xmake/modules/utils/run_script.lua
index a5ab2fa50..82fa8cb0d 100644
--- a/xmake/modules/utils/run_script.lua
+++ b/xmake/modules/utils/run_script.lua
@@ -150,14 +150,19 @@ function _run(script, opt)
         option.set("quiet", true, {force = true})
     end
 
-    local curdir = opt.curdir or os.workingdir()
-    local oldir = os.cd(curdir)
+    local oldir
+    if xmake.in_main_thread() then
+        local curdir = opt.curdir or os.workingdir()
+        oldir = os.cd(curdir)
+    end
     if opt.command then
         _run_commanad(script, _get_args(opt), opt)
     else
         _run_script(script, _get_args(opt), opt)
     end
-    os.cd(oldir)
+    if oldir then
+        os.cd(oldir)
+    end
 
     if opt.quiet then
         option.restore()