From 0f9a73c46e1ed57158461d56662db686b42514e1 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 00:36:15 +0800
Subject: add xmake test

---
 xmake/actions/test/main.lua   | 262 ++++++++++++++++++++++++++++++++++++++++++
 xmake/actions/test/xmake.lua  |  48 ++++++++
 xmake/core/project/target.lua |   1 +
 3 files changed, 311 insertions(+)
 create mode 100644 xmake/actions/test/main.lua
 create mode 100644 xmake/actions/test/xmake.lua

diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
new file mode 100644
index 000000000..70a277a86
--- /dev/null
+++ b/xmake/actions/test/main.lua
@@ -0,0 +1,262 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        main.lua
+--
+
+-- imports
+import("core.base.option")
+import("core.base.task")
+import("core.project.config")
+import("core.base.global")
+import("core.project.project")
+import("core.platform.platform")
+import("devel.debugger")
+import("async.runjobs")
+import("private.action.run.runenvs")
+import("private.service.remote_build.action", {alias = "remote_build_action"})
+
+-- run target
+function _do_run_target(target)
+
+    -- only for binary program
+    if not target:is_binary() then
+        return
+    end
+
+    -- get the run directory of target
+    local rundir = target:rundir()
+
+    -- get the absolute target file path
+    local targetfile = path.absolute(target:targetfile())
+
+    -- build run environments
+    local addenvs, setenvs = runenvs.make(target)
+
+    -- get run arguments
+    local args = table.wrap(option.get("arguments") or target:get("runargs"))
+
+    -- debugging?
+    if option.get("debug") then
+        debugger.run(targetfile, args, {curdir = rundir, addenvs = addenvs, setenvs = setenvs})
+    else
+        local envs = runenvs.join(addenvs, setenvs)
+        os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach"), envs = envs})
+    end
+end
+
+-- run target
+function _on_run_target(target)
+
+    -- build target with rules
+    local done = false
+    for _, r in ipairs(target:orderules()) do
+        local on_run = r:script("run")
+        if on_run then
+            on_run(target)
+            done = true
+        end
+    end
+    if done then return end
+
+    -- do run
+    _do_run_target(target)
+end
+
+-- recursively target add env
+function _add_target_pkgenvs(target, targets_added)
+    if targets_added[target:name()] then
+        return
+    end
+    targets_added[target:name()] = true
+    os.addenvs(target:pkgenvs())
+    for _, dep in ipairs(target:orderdeps()) do
+        _add_target_pkgenvs(dep, targets_added)
+    end
+end
+
+-- find target names matching a specific name
+function _find_matching_target_names(targetname)
+    targetname = targetname:lower()
+    local matching_targetnames = {}
+    for _, target in ipairs(project.ordertargets()) do
+        if target:name():lower():find(targetname, 1, true) then
+            table.insert(matching_targetnames, target:name())
+        end
+    end
+
+    table.sort(matching_targetnames)
+    return matching_targetnames
+end
+
+-- run the given target
+function _run(target)
+
+    -- has been disabled?
+    if not target:is_enabled() then
+        return
+    end
+
+    -- enter the environments of the target packages
+    local oldenvs = os.getenvs()
+    _add_target_pkgenvs(target, {})
+
+    -- the target scripts
+    local scripts =
+    {
+        target:script("run_before")
+    ,   function (target)
+            for _, r in ipairs(target:orderules()) do
+                local before_run = r:script("run_before")
+                if before_run then
+                    before_run(target)
+                end
+            end
+        end
+    ,   target:script("run", _on_run_target)
+    ,   function (target)
+            for _, r in ipairs(target:orderules()) do
+                local after_run = r:script("run_after")
+                if after_run then
+                    after_run(target)
+                end
+            end
+        end
+    ,   target:script("run_after")
+    }
+
+    -- run the target scripts
+    for i = 1, 5 do
+        local script = scripts[i]
+        if script ~= nil then
+            script(target)
+        end
+    end
+
+    -- leave the environments of the target packages
+    os.setenvs(oldenvs)
+end
+
+-- check targets
+function _check_targets(targetname, group_pattern)
+
+    -- get targets
+    local targets = {}
+    if targetname then
+        local target = project.target(targetname)
+        if not target then
+            -- check if the name is part of other target to help
+            local possible_targetnames = _find_matching_target_names(targetname)
+            local errors = targetname .. " is not a valid target name for this project"
+            if #possible_targetnames > 0 then
+                errors = errors .. "\nlist of valid target names close to your input:\n - " .. table.concat(possible_targetnames, '\n - ')
+            end
+            raise(errors)
+        end
+
+        table.insert(targets, target)
+    else
+        for _, target in ipairs(project.ordertargets()) do
+            if target:is_binary() or target:script("run") then
+                local group = target:get("group")
+                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                    table.insert(targets, target)
+                end
+            end
+        end
+    end
+
+    -- filter and check targets with builtin-run script
+    local targetnames = {}
+    for _, target in ipairs(targets) do
+        if target:targetfile() and target:is_enabled() and not target:script("run") then
+            local targetfile = target:targetfile()
+            if targetfile and not os.isfile(targetfile) then
+                table.insert(targetnames, target:name())
+            end
+        end
+    end
+
+    -- there are targets that have not yet been built?
+    if #targetnames > 0 then
+        raise("please run `$xmake build [target]` to build the following targets first:\n  -> " .. table.concat(targetnames, '\n  -> '))
+    end
+end
+
+-- main
+function main()
+
+    -- do action for remote?
+    if remote_build_action.enabled() then
+        return remote_build_action()
+    end
+
+    -- load config first
+    config.load()
+
+    -- Automatically build before running
+    if project.policy("run.autobuild") then
+        -- we need clear the previous config and reload it
+        -- to avoid trigger recheck configs
+        config.clear()
+        task.run("build")
+    end
+
+    -- load targets
+    project.load_targets()
+
+    -- check targets first
+    local targetname
+    local group_pattern = option.get("group")
+    if group_pattern then
+        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
+    else
+        targetname = option.get("target")
+    end
+    _check_targets(targetname, group_pattern)
+
+    -- enter project directory
+    local oldir = os.cd(project.directory())
+
+    -- run the given target?
+    if targetname then
+        _run(project.target(targetname))
+    else
+        local targets = {}
+        for _, target in ipairs(project.ordertargets()) do
+            if target:is_binary() or target:script("run") then
+                local group = target:get("group")
+                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                    table.insert(targets, target)
+                end
+            end
+        end
+        local jobs = tonumber(option.get("jobs") or "1")
+        runjobs("run_targets", function (index)
+            local target = targets[index]
+            if target then
+                _run(target)
+            end
+        end, {total = #targets,
+              comax = jobs,
+              isolate = true})
+    end
+
+    -- leave project directory
+    os.cd(oldir)
+end
+
diff --git a/xmake/actions/test/xmake.lua b/xmake/actions/test/xmake.lua
new file mode 100644
index 000000000..ce9455789
--- /dev/null
+++ b/xmake/actions/test/xmake.lua
@@ -0,0 +1,48 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        xmake.lua
+--
+
+task("test")
+    set_category("action")
+    on_run("main")
+    set_menu {
+        usage = "xmake test [options] [target] [arguments]",
+        description = "Run the project tests.",
+        options = {
+            {'a', "all",        "k",   nil  , "Run all targets."                                                    },
+            {'g', "group",      "kv",  nil  , "Run all targets of the given group. It support path pattern matching.",
+                                              "e.g.",
+                                              "    xmake test -g test",
+                                              "    xmake test -g test_*",
+                                              "    xmake test --group=benchmark/*"                                  },
+            {'w', "workdir",    "kv",  nil  , "Work directory of running targets, default is folder of targetfile",
+                                              "e.g.",
+                                              "    xmake test -w .",
+                                              "    xmake test --workdir=`pwd`"                                      },
+            {'j', "jobs",       "kv", "1",    "Set the number of parallel compilation jobs."                        },
+            {},
+            {nil, "tests",     "vs",   nil  , "The test names. It support pattern matching.",
+                                              "e.g.",
+                                              "    xmake test foo",
+                                              "    xmake test foo_*"                                                }
+        }
+    }
+
+
+
diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua
index 61501648f..b5673a668 100644
--- a/xmake/core/project/target.lua
+++ b/xmake/core/project/target.lua
@@ -2467,6 +2467,7 @@ function target.apis()
             -- target.add_xxx
         ,   "target.add_values"
         ,   "target.add_runenvs"
+        ,   "target.add_tests"
         }
     ,   paths =
         {
-- 
cgit v1.3.1


From 621d6df885bc9ee832221f968ffcb07399d791fa Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 00:41:01 +0800
Subject: add tests for xmake test

---
 tests/actions/test/.gitignore     |  8 ++++++++
 tests/actions/test/src/test_1.cpp | 10 ++++++++++
 tests/actions/test/src/test_2.cpp | 10 ++++++++++
 tests/actions/test/src/test_3.cpp | 10 ++++++++++
 tests/actions/test/src/test_4.cpp | 10 ++++++++++
 tests/actions/test/src/test_5.cpp | 10 ++++++++++
 tests/actions/test/src/test_6.cpp | 10 ++++++++++
 tests/actions/test/src/test_7.cpp | 10 ++++++++++
 tests/actions/test/src/test_8.cpp | 10 ++++++++++
 tests/actions/test/src/test_9.cpp | 10 ++++++++++
 tests/actions/test/test.lua       |  4 ++++
 tests/actions/test/xmake.lua      | 13 +++++++++++++
 xmake/actions/test/main.lua       |  8 --------
 13 files changed, 115 insertions(+), 8 deletions(-)
 create mode 100644 tests/actions/test/.gitignore
 create mode 100644 tests/actions/test/src/test_1.cpp
 create mode 100644 tests/actions/test/src/test_2.cpp
 create mode 100644 tests/actions/test/src/test_3.cpp
 create mode 100644 tests/actions/test/src/test_4.cpp
 create mode 100644 tests/actions/test/src/test_5.cpp
 create mode 100644 tests/actions/test/src/test_6.cpp
 create mode 100644 tests/actions/test/src/test_7.cpp
 create mode 100644 tests/actions/test/src/test_8.cpp
 create mode 100644 tests/actions/test/src/test_9.cpp
 create mode 100644 tests/actions/test/test.lua
 create mode 100644 tests/actions/test/xmake.lua

diff --git a/tests/actions/test/.gitignore b/tests/actions/test/.gitignore
new file mode 100644
index 000000000..152105761
--- /dev/null
+++ b/tests/actions/test/.gitignore
@@ -0,0 +1,8 @@
+# Xmake cache
+.xmake/
+build/
+
+# MacOS Cache
+.DS_Store
+
+
diff --git a/tests/actions/test/src/test_1.cpp b/tests/actions/test/src/test_1.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_1.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_2.cpp b/tests/actions/test/src/test_2.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_2.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_3.cpp b/tests/actions/test/src/test_3.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_3.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_4.cpp b/tests/actions/test/src/test_4.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_4.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_5.cpp b/tests/actions/test/src/test_5.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_5.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_6.cpp b/tests/actions/test/src/test_6.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_6.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_7.cpp b/tests/actions/test/src/test_7.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_7.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_8.cpp b/tests/actions/test/src/test_8.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_8.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/src/test_9.cpp b/tests/actions/test/src/test_9.cpp
new file mode 100644
index 000000000..f454e99f3
--- /dev/null
+++ b/tests/actions/test/src/test_9.cpp
@@ -0,0 +1,10 @@
+#include <iostream>
+
+using namespace std;
+
+int main(int argc, char** argv)
+{
+    char const* arg = argc > 1? argv[1] : "xmake";
+    cout << "hello " << arg << endl;
+    return 0;
+}
diff --git a/tests/actions/test/test.lua b/tests/actions/test/test.lua
new file mode 100644
index 000000000..45ce0b914
--- /dev/null
+++ b/tests/actions/test/test.lua
@@ -0,0 +1,4 @@
+function main(t)
+    os.exec("xmake test")
+end
+
diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
new file mode 100644
index 000000000..b9465566f
--- /dev/null
+++ b/tests/actions/test/xmake.lua
@@ -0,0 +1,13 @@
+add_rules("mode.debug", "mode.release")
+
+for _, file in ipairs(os.files("src/test_*.cpp")) do
+    local name = path.basename(file)
+    target(name)
+        set_kind("binary")
+        add_files("src/" .. name .. ".cpp")
+        add_tests(name)
+        add_tests(name .. "_arg", "foo")
+        add_tests(name .. "_pass_output", "foo", {pass_output = "hello foo"})
+        add_tests(name .. "_fail_output", {fail_output = {"hello .*", "hello xmake"}})
+end
+
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 70a277a86..22748b388 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -208,14 +208,6 @@ function main()
     -- load config first
     config.load()
 
-    -- Automatically build before running
-    if project.policy("run.autobuild") then
-        -- we need clear the previous config and reload it
-        -- to avoid trigger recheck configs
-        config.clear()
-        task.run("build")
-    end
-
     -- load targets
     project.load_targets()
 
-- 
cgit v1.3.1


From e6373510a1fc487dd6652a5bc1cd74a966a8749c Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 00:45:08 +0800
Subject: get tests

---
 :w                           | 273 +++++++++++++++++++++++++++++++++++++++++++
 tests/actions/test/xmake.lua |   3 +-
 xmake/actions/test/main.lua  |  21 +++-
 xmake/actions/test/xmake.lua |   2 +-
 4 files changed, 296 insertions(+), 3 deletions(-)
 create mode 100644 :w

diff --git a/:w b/:w
new file mode 100644
index 000000000..1808a9c89
--- /dev/null
+++ b/:w
@@ -0,0 +1,273 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        main.lua
+--
+
+-- imports
+import("core.base.option")
+import("core.base.task")
+import("core.project.config")
+import("core.base.global")
+import("core.project.project")
+import("core.platform.platform")
+import("devel.debugger")
+import("async.runjobs")
+import("private.action.run.runenvs")
+import("private.service.remote_build.action", {alias = "remote_build_action"})
+
+-- run target
+function _do_run_target(target)
+
+    -- only for binary program
+    if not target:is_binary() then
+        return
+    end
+
+    -- get the run directory of target
+    local rundir = target:rundir()
+
+    -- get the absolute target file path
+    local targetfile = path.absolute(target:targetfile())
+
+    -- build run environments
+    local addenvs, setenvs = runenvs.make(target)
+
+    -- get run arguments
+    local args = table.wrap(option.get("arguments") or target:get("runargs"))
+
+    -- debugging?
+    if option.get("debug") then
+        debugger.run(targetfile, args, {curdir = rundir, addenvs = addenvs, setenvs = setenvs})
+    else
+        local envs = runenvs.join(addenvs, setenvs)
+        os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach"), envs = envs})
+    end
+end
+
+-- run target
+function _on_run_target(target)
+
+    -- build target with rules
+    local done = false
+    for _, r in ipairs(target:orderules()) do
+        local on_run = r:script("run")
+        if on_run then
+            on_run(target)
+            done = true
+        end
+    end
+    if done then return end
+
+    -- do run
+    _do_run_target(target)
+end
+
+-- recursively target add env
+function _add_target_pkgenvs(target, targets_added)
+    if targets_added[target:name()] then
+        return
+    end
+    targets_added[target:name()] = true
+    os.addenvs(target:pkgenvs())
+    for _, dep in ipairs(target:orderdeps()) do
+        _add_target_pkgenvs(dep, targets_added)
+    end
+end
+
+-- find target names matching a specific name
+function _find_matching_target_names(targetname)
+    targetname = targetname:lower()
+    local matching_targetnames = {}
+    for _, target in ipairs(project.ordertargets()) do
+        if target:name():lower():find(targetname, 1, true) then
+            table.insert(matching_targetnames, target:name())
+        end
+    end
+
+    table.sort(matching_targetnames)
+    return matching_targetnames
+end
+
+-- run the given target
+function _run(target)
+
+    -- has been disabled?
+    if not target:is_enabled() then
+        return
+    end
+
+    -- enter the environments of the target packages
+    local oldenvs = os.getenvs()
+    _add_target_pkgenvs(target, {})
+
+    -- the target scripts
+    local scripts =
+    {
+        target:script("run_before")
+    ,   function (target)
+            for _, r in ipairs(target:orderules()) do
+                local before_run = r:script("run_before")
+                if before_run then
+                    before_run(target)
+                end
+            end
+        end
+    ,   target:script("run", _on_run_target)
+    ,   function (target)
+            for _, r in ipairs(target:orderules()) do
+                local after_run = r:script("run_after")
+                if after_run then
+                    after_run(target)
+                end
+            end
+        end
+    ,   target:script("run_after")
+    }
+
+    -- run the target scripts
+    for i = 1, 5 do
+        local script = scripts[i]
+        if script ~= nil then
+            script(target)
+        end
+    end
+
+    -- leave the environments of the target packages
+    os.setenvs(oldenvs)
+end
+
+-- check targets
+function _check_targets(targetname, group_pattern)
+
+    -- get targets
+    local targets = {}
+    if targetname then
+        local target = project.target(targetname)
+        if not target then
+            -- check if the name is part of other target to help
+            local possible_targetnames = _find_matching_target_names(targetname)
+            local errors = targetname .. " is not a valid target name for this project"
+            if #possible_targetnames > 0 then
+                errors = errors .. "\nlist of valid target names close to your input:\n - " .. table.concat(possible_targetnames, '\n - ')
+            end
+            raise(errors)
+        end
+
+        table.insert(targets, target)
+    else
+        for _, target in ipairs(project.ordertargets()) do
+            if target:is_binary() or target:script("run") then
+                local group = target:get("group")
+                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                    table.insert(targets, target)
+                end
+            end
+        end
+    end
+
+    -- filter and check targets with builtin-run script
+    local targetnames = {}
+    for _, target in ipairs(targets) do
+        if target:targetfile() and target:is_enabled() and not target:script("run") then
+            local targetfile = target:targetfile()
+            if targetfile and not os.isfile(targetfile) then
+                table.insert(targetnames, target:name())
+            end
+        end
+    end
+
+    -- there are targets that have not yet been built?
+    if #targetnames > 0 then
+        raise("please run `$xmake build [target]` to build the following targets first:\n  -> " .. table.concat(targetnames, '\n  -> '))
+    end
+end
+
+-- main
+function main()
+
+    -- do action for remote?
+    if remote_build_action.enabled() then
+        return remote_build_action()
+    end
+
+    -- load config first
+    config.load()
+
+    -- load targets
+    project.load_targets()
+
+    -- get tests
+    local tests = {}
+    for _, target in ipairs(project.ordertargets()) do
+        if target:is_binary() or target:script("run") then
+        print(target:name())
+            for _, test in ipairs(target:get("tests")) do
+                print(test)
+
+                --table.insert(tests, )
+            end
+            --[[
+            local group = target:get("group")
+            if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                table.insert(targets, target)
+            end]]
+        end
+    end
+
+    --[[
+    -- check targets first
+    local targetname
+    local group_pattern = option.get("group")
+    if group_pattern then
+        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
+    else
+        targetname = option.get("target")
+    end
+    _check_targets(targetname, group_pattern)
+
+    -- enter project directory
+    local oldir = os.cd(project.directory())
+
+    -- run the given target?
+    if targetname then
+        _run(project.target(targetname))
+    else
+        local targets = {}
+        for _, target in ipairs(project.ordertargets()) do
+            if target:is_binary() or target:script("run") then
+                local group = target:get("group")
+                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                    table.insert(targets, target)
+                end
+            end
+        end
+        local jobs = tonumber(option.get("jobs") or "1")
+        runjobs("run_targets", function (index)
+            local target = targets[index]
+            if target then
+                _run(target)
+            end
+        end, {total = #targets,
+              comax = jobs,
+              isolate = true})
+    end
+
+    -- leave project directory
+    os.cd(oldir)]]
+end
+
diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index b9465566f..f02f50c0c 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -4,9 +4,10 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
     local name = path.basename(file)
     target(name)
         set_kind("binary")
+        set_default(false)
         add_files("src/" .. name .. ".cpp")
         add_tests(name)
-        add_tests(name .. "_arg", "foo")
+        add_tests(name .. "_arg", "foo", "bar")
         add_tests(name .. "_pass_output", "foo", {pass_output = "hello foo"})
         add_tests(name .. "_fail_output", {fail_output = {"hello .*", "hello xmake"}})
 end
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 22748b388..667ae2a0e 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -211,6 +211,25 @@ function main()
     -- load targets
     project.load_targets()
 
+    -- get tests
+    local tests = {}
+    for _, target in ipairs(project.ordertargets()) do
+        if target:is_binary() or target:script("run") then
+            for name, argv in pairs(target:get("tests")) do
+                local extra = target:extraconf("tests", name)
+                print(name, extra)
+                tests[name] = table.join({argv = table.wrap(argv)}, extra)
+            end
+            --[[
+            local group = target:get("group")
+            if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                table.insert(targets, target)
+            end]]
+        end
+    end
+--    print(tests)
+
+    --[[
     -- check targets first
     local targetname
     local group_pattern = option.get("group")
@@ -249,6 +268,6 @@ function main()
     end
 
     -- leave project directory
-    os.cd(oldir)
+    os.cd(oldir)]]
 end
 
diff --git a/xmake/actions/test/xmake.lua b/xmake/actions/test/xmake.lua
index ce9455789..4a48f4621 100644
--- a/xmake/actions/test/xmake.lua
+++ b/xmake/actions/test/xmake.lua
@@ -37,7 +37,7 @@ task("test")
                                               "    xmake test --workdir=`pwd`"                                      },
             {'j', "jobs",       "kv", "1",    "Set the number of parallel compilation jobs."                        },
             {},
-            {nil, "tests",     "vs",   nil  , "The test names. It support pattern matching.",
+            {nil, "tests",     "vs",  nil   , "The test names. It support pattern matching.",
                                               "e.g.",
                                               "    xmake test foo",
                                               "    xmake test foo_*"                                                }
-- 
cgit v1.3.1


From c12127b37c2d967a928cb039187d1ea65d958803 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 00:45:09 +0800
Subject: get tests

---
 :w | 273 ---------------------------------------------------------------------
 1 file changed, 273 deletions(-)
 delete mode 100644 :w

diff --git a/:w b/:w
deleted file mode 100644
index 1808a9c89..000000000
--- a/:w
+++ /dev/null
@@ -1,273 +0,0 @@
---!A cross-platform build utility based on Lua
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
--- Copyright (C) 2015-present, TBOOX Open Source Group.
---
--- @author      ruki
--- @file        main.lua
---
-
--- imports
-import("core.base.option")
-import("core.base.task")
-import("core.project.config")
-import("core.base.global")
-import("core.project.project")
-import("core.platform.platform")
-import("devel.debugger")
-import("async.runjobs")
-import("private.action.run.runenvs")
-import("private.service.remote_build.action", {alias = "remote_build_action"})
-
--- run target
-function _do_run_target(target)
-
-    -- only for binary program
-    if not target:is_binary() then
-        return
-    end
-
-    -- get the run directory of target
-    local rundir = target:rundir()
-
-    -- get the absolute target file path
-    local targetfile = path.absolute(target:targetfile())
-
-    -- build run environments
-    local addenvs, setenvs = runenvs.make(target)
-
-    -- get run arguments
-    local args = table.wrap(option.get("arguments") or target:get("runargs"))
-
-    -- debugging?
-    if option.get("debug") then
-        debugger.run(targetfile, args, {curdir = rundir, addenvs = addenvs, setenvs = setenvs})
-    else
-        local envs = runenvs.join(addenvs, setenvs)
-        os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach"), envs = envs})
-    end
-end
-
--- run target
-function _on_run_target(target)
-
-    -- build target with rules
-    local done = false
-    for _, r in ipairs(target:orderules()) do
-        local on_run = r:script("run")
-        if on_run then
-            on_run(target)
-            done = true
-        end
-    end
-    if done then return end
-
-    -- do run
-    _do_run_target(target)
-end
-
--- recursively target add env
-function _add_target_pkgenvs(target, targets_added)
-    if targets_added[target:name()] then
-        return
-    end
-    targets_added[target:name()] = true
-    os.addenvs(target:pkgenvs())
-    for _, dep in ipairs(target:orderdeps()) do
-        _add_target_pkgenvs(dep, targets_added)
-    end
-end
-
--- find target names matching a specific name
-function _find_matching_target_names(targetname)
-    targetname = targetname:lower()
-    local matching_targetnames = {}
-    for _, target in ipairs(project.ordertargets()) do
-        if target:name():lower():find(targetname, 1, true) then
-            table.insert(matching_targetnames, target:name())
-        end
-    end
-
-    table.sort(matching_targetnames)
-    return matching_targetnames
-end
-
--- run the given target
-function _run(target)
-
-    -- has been disabled?
-    if not target:is_enabled() then
-        return
-    end
-
-    -- enter the environments of the target packages
-    local oldenvs = os.getenvs()
-    _add_target_pkgenvs(target, {})
-
-    -- the target scripts
-    local scripts =
-    {
-        target:script("run_before")
-    ,   function (target)
-            for _, r in ipairs(target:orderules()) do
-                local before_run = r:script("run_before")
-                if before_run then
-                    before_run(target)
-                end
-            end
-        end
-    ,   target:script("run", _on_run_target)
-    ,   function (target)
-            for _, r in ipairs(target:orderules()) do
-                local after_run = r:script("run_after")
-                if after_run then
-                    after_run(target)
-                end
-            end
-        end
-    ,   target:script("run_after")
-    }
-
-    -- run the target scripts
-    for i = 1, 5 do
-        local script = scripts[i]
-        if script ~= nil then
-            script(target)
-        end
-    end
-
-    -- leave the environments of the target packages
-    os.setenvs(oldenvs)
-end
-
--- check targets
-function _check_targets(targetname, group_pattern)
-
-    -- get targets
-    local targets = {}
-    if targetname then
-        local target = project.target(targetname)
-        if not target then
-            -- check if the name is part of other target to help
-            local possible_targetnames = _find_matching_target_names(targetname)
-            local errors = targetname .. " is not a valid target name for this project"
-            if #possible_targetnames > 0 then
-                errors = errors .. "\nlist of valid target names close to your input:\n - " .. table.concat(possible_targetnames, '\n - ')
-            end
-            raise(errors)
-        end
-
-        table.insert(targets, target)
-    else
-        for _, target in ipairs(project.ordertargets()) do
-            if target:is_binary() or target:script("run") then
-                local group = target:get("group")
-                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    table.insert(targets, target)
-                end
-            end
-        end
-    end
-
-    -- filter and check targets with builtin-run script
-    local targetnames = {}
-    for _, target in ipairs(targets) do
-        if target:targetfile() and target:is_enabled() and not target:script("run") then
-            local targetfile = target:targetfile()
-            if targetfile and not os.isfile(targetfile) then
-                table.insert(targetnames, target:name())
-            end
-        end
-    end
-
-    -- there are targets that have not yet been built?
-    if #targetnames > 0 then
-        raise("please run `$xmake build [target]` to build the following targets first:\n  -> " .. table.concat(targetnames, '\n  -> '))
-    end
-end
-
--- main
-function main()
-
-    -- do action for remote?
-    if remote_build_action.enabled() then
-        return remote_build_action()
-    end
-
-    -- load config first
-    config.load()
-
-    -- load targets
-    project.load_targets()
-
-    -- get tests
-    local tests = {}
-    for _, target in ipairs(project.ordertargets()) do
-        if target:is_binary() or target:script("run") then
-        print(target:name())
-            for _, test in ipairs(target:get("tests")) do
-                print(test)
-
-                --table.insert(tests, )
-            end
-            --[[
-            local group = target:get("group")
-            if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                table.insert(targets, target)
-            end]]
-        end
-    end
-
-    --[[
-    -- check targets first
-    local targetname
-    local group_pattern = option.get("group")
-    if group_pattern then
-        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
-    else
-        targetname = option.get("target")
-    end
-    _check_targets(targetname, group_pattern)
-
-    -- enter project directory
-    local oldir = os.cd(project.directory())
-
-    -- run the given target?
-    if targetname then
-        _run(project.target(targetname))
-    else
-        local targets = {}
-        for _, target in ipairs(project.ordertargets()) do
-            if target:is_binary() or target:script("run") then
-                local group = target:get("group")
-                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    table.insert(targets, target)
-                end
-            end
-        end
-        local jobs = tonumber(option.get("jobs") or "1")
-        runjobs("run_targets", function (index)
-            local target = targets[index]
-            if target then
-                _run(target)
-            end
-        end, {total = #targets,
-              comax = jobs,
-              isolate = true})
-    end
-
-    -- leave project directory
-    os.cd(oldir)]]
-end
-
-- 
cgit v1.3.1


From ebb301cf17c2aa25b1f698ce2e5bd2eb753a9bc5 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 10:21:33 +0800
Subject: auto build targets for tests

---
 tests/actions/test/xmake.lua  |   4 +-
 xmake/actions/build/build.lua |  26 +++----
 xmake/actions/build/check.lua |   8 ++-
 xmake/actions/build/main.lua  |  96 ++++++++++++++------------
 xmake/actions/test/main.lua   | 156 +++++++++++++++---------------------------
 xmake/core/project/target.lua |   2 +-
 6 files changed, 128 insertions(+), 164 deletions(-)

diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index f02f50c0c..fac51a018 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -7,8 +7,8 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
         set_default(false)
         add_files("src/" .. name .. ".cpp")
         add_tests(name)
-        add_tests(name .. "_arg", "foo", "bar")
-        add_tests(name .. "_pass_output", "foo", {pass_output = "hello foo"})
+        add_tests(name .. "_args", {arguments = {"foo", "bar"}})
+        add_tests(name .. "_pass_output", {arguments = "foo", pass_output = "hello foo"})
         add_tests(name .. "_fail_output", {fail_output = {"hello .*", "hello xmake"}})
 end
 
diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua
index 5cea43185..bb0b61cc1 100644
--- a/xmake/actions/build/build.lua
+++ b/xmake/actions/build/build.lua
@@ -229,19 +229,21 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target)
 end
 
 -- get batch jobs, @note we need to export it for private.diagnosis.dump_buildjobs
-function get_batchjobs(targetname, group_pattern)
+function get_batchjobs(targetnames, group_pattern)
 
     -- get root targets
     local targets_root = {}
-    if targetname then
-        local target = project.target(targetname)
-        if target then
-            table.insert(targets_root, target)
-            if option.get("rebuild") then
-                target:data_set("rebuilt", true)
-                if not option.get("shallow") then
-                    for _, dep in ipairs(target:orderdeps()) do
-                        dep:data_set("rebuilt", true)
+    if targetnames then
+        for _, targetname in ipairs(table.wrap(targetnames)) do
+            local target = project.target(targetname)
+            if target then
+                table.insert(targets_root, target)
+                if option.get("rebuild") then
+                    target:data_set("rebuilt", true)
+                    if not option.get("shallow") then
+                        for _, dep in ipairs(target:orderdeps()) do
+                            dep:data_set("rebuilt", true)
+                        end
                     end
                 end
             end
@@ -280,7 +282,7 @@ function get_batchjobs(targetname, group_pattern)
 end
 
 -- the main entry
-function main(targetname, group_pattern)
+function main(targetnames, group_pattern)
 
     -- enable distcc?
     local distcc
@@ -289,7 +291,7 @@ function main(targetname, group_pattern)
     end
 
     -- build all jobs
-    local batchjobs = get_batchjobs(targetname, group_pattern)
+    local batchjobs = get_batchjobs(targetnames, group_pattern)
     if batchjobs and batchjobs:size() > 0 then
         local curdir = os.curdir()
         runjobs("build", batchjobs, {on_exit = function (errors)
diff --git a/xmake/actions/build/check.lua b/xmake/actions/build/check.lua
index e26e4f0d0..f468acfb1 100644
--- a/xmake/actions/build/check.lua
+++ b/xmake/actions/build/check.lua
@@ -44,13 +44,15 @@ function _show(str, opt)
     end
 end
 
-function main(targetname, opt)
+function main(targetnames, opt)
     opt = opt or {}
 
     -- get targets
     local targets = {}
-    if targetname then
-        table.insert(targets, project.target(targetname))
+    if targetnames then
+        for _, targetname in ipairs(table.wrap(targetnames)) do
+            table.insert(targets, project.target(targetname))
+        end
     else
         for _, target in pairs(project.targets()) do
             if target:is_enabled() then
diff --git a/xmake/actions/build/main.lua b/xmake/actions/build/main.lua
index 58456ba70..43897578a 100644
--- a/xmake/actions/build/main.lua
+++ b/xmake/actions/build/main.lua
@@ -129,61 +129,26 @@ function _on_exit(ok, errors)
     end
 end
 
--- main
-function main()
-
-    -- try building it using third-party buildsystem if xmake.lua not exists
-    if not os.isfile(project.rootfile()) and _try_build() then
-        return
-    end
-
-    -- post statistics before locking project
-    statistics.post()
-
-    -- do action for remote?
-    if remote_build_action.enabled() then
-        return remote_build_action()
-    end
-
-    -- lock the whole project
-    project.lock()
-
-    -- config it first
-    local targetname
-    local group_pattern = option.get("group")
-    if group_pattern then
-        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
-    else
-        targetname = option.get("target")
-    end
-    task.run("config", {}, {disable_dump = true})
-
-    -- enter project directory
-    local oldir = os.cd(project.directory())
-
-    -- clean up temporary files once a day
-    cleaner.cleanup()
+-- build targets
+function build_targets(targetnames, opt)
+    opt = opt or {}
 
     -- register exit callbacks
     os.atexit(_on_exit)
 
-    local build_time
+    local group_pattern = opt.group_pattern
     try
     {
         function ()
-            local time = os.mclock()
 
             -- do rules before building
             _do_project_rules("build_before")
 
             -- do build
-            _do_build(targetname, group_pattern)
+            _do_build(targetnames, group_pattern)
 
             -- do check
-            check_targets(targetname, {build = true})
-
-            -- get build time
-            build_time = os.mclock() - time
+            check_targets(targetnames, {build = true})
 
             -- dump cache stats
             if option.get("diagnosis") then
@@ -206,8 +171,9 @@ function main()
                     raise(errors)
                 elseif group_pattern then
                     raise("build targets with group(%s) failed!", group_pattern)
-                elseif targetname then
-                    raise("build target: %s failed!", targetname)
+                elseif targetnames then
+                    targetnames = table.wrap(targetnames)
+                    raise("build target: %s failed!", table.concat(targetnames, ", "))
                 else
                     raise("build target failed!")
                 end
@@ -217,13 +183,53 @@ function main()
 
     -- do rules after building
     _do_project_rules("build_after")
+end
 
-    -- unlock the whole project
-    project.unlock()
+function main()
+
+    -- try building it using third-party buildsystem if xmake.lua not exists
+    if not os.isfile(project.rootfile()) and _try_build() then
+        return
+    end
+
+    -- post statistics before locking project
+    statistics.post()
+
+    -- do action for remote?
+    if remote_build_action.enabled() then
+        return remote_build_action()
+    end
+
+    -- lock the whole project
+    project.lock()
+
+    -- config it first
+    local targetname
+    local group_pattern = option.get("group")
+    if group_pattern then
+        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
+    else
+        targetname = option.get("target")
+    end
+    task.run("config", {}, {disable_dump = true})
+
+    -- enter project directory
+    local oldir = os.cd(project.directory())
+
+    -- clean up temporary files once a day
+    cleaner.cleanup()
+
+    -- build targets
+    local build_time = os.mclock()
+    build_targets(targetname, {group_pattern = group_pattern})
+    build_time = os.mclock() - build_time
 
     -- leave project directory
     os.cd(oldir)
 
+    -- unlock the whole project
+    project.unlock()
+
     -- trace
     local str = ""
     if build_time then
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 667ae2a0e..3c113d661 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -29,6 +29,7 @@ import("devel.debugger")
 import("async.runjobs")
 import("private.action.run.runenvs")
 import("private.service.remote_build.action", {alias = "remote_build_action"})
+import("actions.build.main", {rootdir = os.programdir(), alias = "build_action"})
 
 -- run target
 function _do_run_target(target)
@@ -89,20 +90,6 @@ function _add_target_pkgenvs(target, targets_added)
     end
 end
 
--- find target names matching a specific name
-function _find_matching_target_names(targetname)
-    targetname = targetname:lower()
-    local matching_targetnames = {}
-    for _, target in ipairs(project.ordertargets()) do
-        if target:name():lower():find(targetname, 1, true) then
-            table.insert(matching_targetnames, target:name())
-        end
-    end
-
-    table.sort(matching_targetnames)
-    return matching_targetnames
-end
-
 -- run the given target
 function _run(target)
 
@@ -151,53 +138,10 @@ function _run(target)
     os.setenvs(oldenvs)
 end
 
--- check targets
-function _check_targets(targetname, group_pattern)
-
-    -- get targets
-    local targets = {}
-    if targetname then
-        local target = project.target(targetname)
-        if not target then
-            -- check if the name is part of other target to help
-            local possible_targetnames = _find_matching_target_names(targetname)
-            local errors = targetname .. " is not a valid target name for this project"
-            if #possible_targetnames > 0 then
-                errors = errors .. "\nlist of valid target names close to your input:\n - " .. table.concat(possible_targetnames, '\n - ')
-            end
-            raise(errors)
-        end
-
-        table.insert(targets, target)
-    else
-        for _, target in ipairs(project.ordertargets()) do
-            if target:is_binary() or target:script("run") then
-                local group = target:get("group")
-                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    table.insert(targets, target)
-                end
-            end
-        end
-    end
-
-    -- filter and check targets with builtin-run script
-    local targetnames = {}
-    for _, target in ipairs(targets) do
-        if target:targetfile() and target:is_enabled() and not target:script("run") then
-            local targetfile = target:targetfile()
-            if targetfile and not os.isfile(targetfile) then
-                table.insert(targetnames, target:name())
-            end
-        end
-    end
-
-    -- there are targets that have not yet been built?
-    if #targetnames > 0 then
-        raise("please run `$xmake build [target]` to build the following targets first:\n  -> " .. table.concat(targetnames, '\n  -> '))
-    end
+-- run tests
+function _run_tests(tests)
 end
 
--- main
 function main()
 
     -- do action for remote?
@@ -205,6 +149,9 @@ function main()
         return remote_build_action()
     end
 
+    -- lock the whole project
+    project.lock()
+
     -- load config first
     config.load()
 
@@ -213,61 +160,68 @@ function main()
 
     -- get tests
     local tests = {}
+    local group_pattern = option.get("group")
+    if group_pattern then
+        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
+    end
     for _, target in ipairs(project.ordertargets()) do
         if target:is_binary() or target:script("run") then
-            for name, argv in pairs(target:get("tests")) do
+            for _, name in ipairs(target:get("tests")) do
+                local info = {target = target}
                 local extra = target:extraconf("tests", name)
-                print(name, extra)
-                tests[name] = table.join({argv = table.wrap(argv)}, extra)
+                if extra then
+                    table.join2(info, extra)
+                end
+                if not info.group then
+                    info.group = target:get("group")
+                end
+                if not info.rundir then
+                    info.rundir = target:rundir()
+                end
+                if not info.runenvs then
+                    local addenvs, setenvs = runenvs.make(target)
+                    local envs = runenvs.join(addenvs, setenvs)
+                    info.runenvs = envs
+                end
+
+                local group = info.group
+                if (not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                    tests[name] = info
+                end
             end
-            --[[
-            local group = target:get("group")
-            if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                table.insert(targets, target)
-            end]]
         end
     end
---    print(tests)
-
-    --[[
-    -- check targets first
-    local targetname
-    local group_pattern = option.get("group")
-    if group_pattern then
-        group_pattern = "^" .. path.pattern(group_pattern) .. "$"
-    else
-        targetname = option.get("target")
+    local test_patterns = option.get("tests")
+    if test_patterns then
+        local tests_new = {}
+        for _, pattern in ipairs(test_patterns) do
+            pattern = "^" .. path.pattern(pattern) .. "$"
+            for name, info in pairs(tests) do
+                if name:match(pattern) then
+                    tests_new[name] = info
+                end
+            end
+        end
+        tests = tests_new
     end
-    _check_targets(targetname, group_pattern)
 
     -- enter project directory
     local oldir = os.cd(project.directory())
 
-    -- run the given target?
-    if targetname then
-        _run(project.target(targetname))
-    else
-        local targets = {}
-        for _, target in ipairs(project.ordertargets()) do
-            if target:is_binary() or target:script("run") then
-                local group = target:get("group")
-                if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    table.insert(targets, target)
-                end
-            end
-        end
-        local jobs = tonumber(option.get("jobs") or "1")
-        runjobs("run_targets", function (index)
-            local target = targets[index]
-            if target then
-                _run(target)
-            end
-        end, {total = #targets,
-              comax = jobs,
-              isolate = true})
+    -- build targets with the given tests first
+    local targetnames = {}
+    for _, info in table.orderpairs(tests) do
+        table.insert(targetnames, info.target:name())
     end
+    build_action.build_targets(targetnames)
+
+    -- run tests
+    _run_tests(tests)
 
     -- leave project directory
-    os.cd(oldir)]]
+    os.cd(oldir)
+
+    -- unlock the whole project
+    project.unlock()
 end
 
diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua
index b5673a668..56b7cb7b8 100644
--- a/xmake/core/project/target.lua
+++ b/xmake/core/project/target.lua
@@ -2455,6 +2455,7 @@ function target.apis()
         ,   "target.add_languages"
         ,   "target.add_vectorexts"
         ,   "target.add_toolchains"
+        ,   "target.add_tests"
         }
     ,   keyvalues =
         {
@@ -2467,7 +2468,6 @@ function target.apis()
             -- target.add_xxx
         ,   "target.add_values"
         ,   "target.add_runenvs"
-        ,   "target.add_tests"
         }
     ,   paths =
         {
-- 
cgit v1.3.1


From a7f4569b042b3d909a37f37b192d3c0961c48621 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 11:44:05 +0800
Subject: match test output

---
 tests/actions/test/xmake.lua  |   6 +-
 xmake/actions/run/main.lua    |   4 +-
 xmake/actions/test/main.lua   | 193 +++++++++++++++++++++++++++---------------
 xmake/core/project/rule.lua   |   3 +
 xmake/core/project/target.lua |   3 +
 5 files changed, 137 insertions(+), 72 deletions(-)

diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index fac51a018..0ca6369a0 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -7,8 +7,8 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
         set_default(false)
         add_files("src/" .. name .. ".cpp")
         add_tests(name)
-        add_tests(name .. "_args", {arguments = {"foo", "bar"}})
-        add_tests(name .. "_pass_output", {arguments = "foo", pass_output = "hello foo"})
-        add_tests(name .. "_fail_output", {fail_output = {"hello .*", "hello xmake"}})
+        add_tests(name .. "_args", {runargs = {"foo", "bar"}})
+        add_tests(name .. "_pass_output", {runargs = "foo", pass_outputs = "hello foo"})
+        add_tests(name .. "_fail_output", {fail_outputs = {"hello .*", "hello xmake"}})
 end
 
diff --git a/xmake/actions/run/main.lua b/xmake/actions/run/main.lua
index 70a277a86..217f87c66 100644
--- a/xmake/actions/run/main.lua
+++ b/xmake/actions/run/main.lua
@@ -44,7 +44,7 @@ function _do_run_target(target)
     -- get the absolute target file path
     local targetfile = path.absolute(target:targetfile())
 
-    -- build run environments
+    -- get the run environments
     local addenvs, setenvs = runenvs.make(target)
 
     -- get run arguments
@@ -77,7 +77,7 @@ function _on_run_target(target)
     _do_run_target(target)
 end
 
--- recursively target add env
+-- recursively add target envs
 function _add_target_pkgenvs(target, targets_added)
     if targets_added[target:name()] then
         return
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 3c113d661..fe54119cf 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -25,60 +25,92 @@ import("core.project.config")
 import("core.base.global")
 import("core.project.project")
 import("core.platform.platform")
-import("devel.debugger")
 import("async.runjobs")
 import("private.action.run.runenvs")
 import("private.service.remote_build.action", {alias = "remote_build_action"})
 import("actions.build.main", {rootdir = os.programdir(), alias = "build_action"})
 
--- run target
-function _do_run_target(target)
+-- test target
+function _do_test_target(target, opt)
+    opt = opt or {}
 
-    -- only for binary program
-    if not target:is_binary() then
-        return
+    -- get run environments
+    local envs = opt.runenvs
+    if not envs then
+        local addenvs, setenvs = runenvs.make(target)
+        envs = runenvs.join(addenvs, setenvs)
     end
 
-    -- get the run directory of target
-    local rundir = target:rundir()
-
-    -- get the absolute target file path
+    -- run test
+    local outdata
+    local rundir = opt.rundir or target:rundir()
     local targetfile = path.absolute(target:targetfile())
+    local runargs = table.wrap(opt.runargs or target:get("runargs"))
+    local ok = try {
+        function ()
+            outdata = os.iorunv(targetfile, runargs, {curdir = rundir, envs = envs})
+            return true
+        end
+    }
 
-    -- build run environments
-    local addenvs, setenvs = runenvs.make(target)
-
-    -- get run arguments
-    local args = table.wrap(option.get("arguments") or target:get("runargs"))
-
-    -- debugging?
-    if option.get("debug") then
-        debugger.run(targetfile, args, {curdir = rundir, addenvs = addenvs, setenvs = setenvs})
-    else
-        local envs = runenvs.join(addenvs, setenvs)
-        os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach"), envs = envs})
+    if ok then
+        local passed
+        outdata = outdata or ""
+        for _, pass_output in ipairs(opt.pass_outputs) do
+            if opt.plain then
+                if pass_output == outdata then
+                    passed = true
+                    break
+                end
+            else
+                if outdata:match("^" .. pass_output .. "$") then
+                    passed = true
+                    break
+                end
+            end
+        end
+        for _, fail_output in ipairs(opt.fail_outputs) do
+            if opt.plain then
+                if fail_output == outdata then
+                    passed = false
+                    break
+                end
+            else
+                if outdata:match("^" .. fail_output .. "$") then
+                    passed = false
+                    break
+                end
+            end
+        end
+        if passed == nil then
+            passed = true
+        end
+        return passed
     end
 end
 
--- run target
-function _on_run_target(target)
+-- test target
+function _on_test_target(target, opt)
 
     -- build target with rules
+    local passed
     local done = false
     for _, r in ipairs(target:orderules()) do
-        local on_run = r:script("run")
-        if on_run then
-            on_run(target)
+        local on_test = r:script("test")
+        if on_test then
+            passed = on_test(target, opt)
             done = true
         end
     end
-    if done then return end
+    if done then
+        return passed
+    end
 
-    -- do run
-    _do_run_target(target)
+    -- do test
+    return _do_test_target(target, opt)
 end
 
--- recursively target add env
+-- recursively add target envs
 function _add_target_pkgenvs(target, targets_added)
     if targets_added[target:name()] then
         return
@@ -90,13 +122,12 @@ function _add_target_pkgenvs(target, targets_added)
     end
 end
 
--- run the given target
-function _run(target)
+-- run the given test
+function _run_test(test)
 
-    -- has been disabled?
-    if not target:is_enabled() then
-        return
-    end
+    -- this target has been disabled?
+    local target = test.target
+    test.target = nil
 
     -- enter the environments of the target packages
     local oldenvs = os.getenvs()
@@ -105,41 +136,77 @@ function _run(target)
     -- the target scripts
     local scripts =
     {
-        target:script("run_before")
-    ,   function (target)
+        target:script("test_before")
+    ,   function (target, opt)
             for _, r in ipairs(target:orderules()) do
-                local before_run = r:script("run_before")
-                if before_run then
-                    before_run(target)
+                local before_test = r:script("test_before")
+                if before_test then
+                    before_test(target, opt)
                 end
             end
         end
-    ,   target:script("run", _on_run_target)
-    ,   function (target)
+    ,   target:script("test", _on_test_target)
+    ,   function (target, opt)
             for _, r in ipairs(target:orderules()) do
-                local after_run = r:script("run_after")
-                if after_run then
-                    after_run(target)
+                local after_test = r:script("test_after")
+                if after_test then
+                    after_test(target, opt)
                 end
             end
         end
-    ,   target:script("run_after")
+    ,   target:script("test_after")
     }
 
     -- run the target scripts
+    local passed
     for i = 1, 5 do
         local script = scripts[i]
         if script ~= nil then
-            script(target)
+            local ok = script(target, test)
+            if i == 3 then
+                passed = ok
+            end
         end
     end
 
     -- leave the environments of the target packages
     os.setenvs(oldenvs)
+    return passed
 end
 
 -- run tests
 function _run_tests(tests)
+    local ordertests = {}
+    for name, testinfo in table.orderpairs(tests) do
+        table.insert(ordertests, testinfo)
+    end
+    if #ordertests == 0 then
+        print("nothing to test")
+        return
+    end
+
+    -- do test
+    local spent = os.mclock()
+    print("running tests ...")
+    local report = {passed = 0, total = #ordertests}
+    local jobs = tonumber(option.get("jobs") or "1")
+    runjobs("run_tests", function (index)
+        local testinfo = ordertests[index]
+        if testinfo then
+            local passed = _run_test(testinfo)
+            if passed then
+                report.passed = report.passed + 1
+            end
+        end
+    end, {total = #ordertests,
+          comax = jobs,
+          isolate = true})
+
+    -- generate report
+    spent = os.mclock() - spent
+    local passed_rate = math.floor(report.passed * 100 / report.total)
+    cprint("${color.success}%3d%%${clear} tests passed, ${color.failure}%d${clear} tests failed out of ${bright}%d${clear}, spent ${bright}%0.3fs",
+        passed_rate, report.total - report.passed, report.total, spent / 1000)
 end
 
 function main()
@@ -167,26 +234,18 @@ function main()
     for _, target in ipairs(project.ordertargets()) do
         if target:is_binary() or target:script("run") then
             for _, name in ipairs(target:get("tests")) do
-                local info = {target = target}
+                local testinfo = {name = name, target = target}
                 local extra = target:extraconf("tests", name)
                 if extra then
-                    table.join2(info, extra)
-                end
-                if not info.group then
-                    info.group = target:get("group")
-                end
-                if not info.rundir then
-                    info.rundir = target:rundir()
+                    table.join2(testinfo, extra)
                 end
-                if not info.runenvs then
-                    local addenvs, setenvs = runenvs.make(target)
-                    local envs = runenvs.join(addenvs, setenvs)
-                    info.runenvs = envs
+                if not testinfo.group then
+                    testinfo.group = target:get("group")
                 end
 
-                local group = info.group
+                local group = testinfo.group
                 if (not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    tests[name] = info
+                    tests[name] = testinfo
                 end
             end
         end
@@ -196,9 +255,9 @@ function main()
         local tests_new = {}
         for _, pattern in ipairs(test_patterns) do
             pattern = "^" .. path.pattern(pattern) .. "$"
-            for name, info in pairs(tests) do
+            for name, testinfo in pairs(tests) do
                 if name:match(pattern) then
-                    tests_new[name] = info
+                    tests_new[name] = testinfo
                 end
             end
         end
@@ -210,8 +269,8 @@ function main()
 
     -- build targets with the given tests first
     local targetnames = {}
-    for _, info in table.orderpairs(tests) do
-        table.insert(targetnames, info.target:name())
+    for _, testinfo in table.orderpairs(tests) do
+        table.insert(targetnames, testinfo.target:name())
     end
     build_action.build_targets(targetnames)
 
diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua
index 4d2a058a0..57510b45a 100644
--- a/xmake/core/project/rule.lua
+++ b/xmake/core/project/rule.lua
@@ -308,6 +308,7 @@ function rule.apis()
         {
             -- rule.on_xxx
             "rule.on_run"
+        ,   "rule.on_test"
         ,   "rule.on_load"
         ,   "rule.on_config"
         ,   "rule.on_link"
@@ -324,6 +325,7 @@ function rule.apis()
         ,   "rule.on_buildcmd_files"
             -- rule.before_xxx
         ,   "rule.before_run"
+        ,   "rule.before_test"
         ,   "rule.before_load"
         ,   "rule.before_link"
         ,   "rule.before_build"
@@ -339,6 +341,7 @@ function rule.apis()
         ,   "rule.before_buildcmd_files"
             -- rule.after_xxx
         ,   "rule.after_run"
+        ,   "rule.after_test"
         ,   "rule.after_load"
         ,   "rule.after_link"
         ,   "rule.after_build"
diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua
index 56b7cb7b8..607bc68cc 100644
--- a/xmake/core/project/target.lua
+++ b/xmake/core/project/target.lua
@@ -2494,6 +2494,7 @@ function target.apis()
         {
             -- target.on_xxx
             "target.on_run"
+        ,   "target.on_test"
         ,   "target.on_load"
         ,   "target.on_config"
         ,   "target.on_link"
@@ -2506,6 +2507,7 @@ function target.apis()
         ,   "target.on_uninstall"
             -- target.before_xxx
         ,   "target.before_run"
+        ,   "target.before_test"
         ,   "target.before_link"
         ,   "target.before_build"
         ,   "target.before_build_file"
@@ -2516,6 +2518,7 @@ function target.apis()
         ,   "target.before_uninstall"
             -- target.after_xxx
         ,   "target.after_run"
+        ,   "target.after_test"
         ,   "target.after_load"
         ,   "target.after_link"
         ,   "target.after_build"
-- 
cgit v1.3.1


From 943e49ac18086207b4371d668d99fd8e8995042c Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 12:52:37 +0800
Subject: format test output

---
 tests/actions/test/xmake.lua |  8 ++++----
 xmake/actions/test/main.lua  | 27 ++++++++++++++++++++-------
 xmake/actions/test/xmake.lua |  3 ++-
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index 0ca6369a0..a60d964a2 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -6,9 +6,9 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
         set_kind("binary")
         set_default(false)
         add_files("src/" .. name .. ".cpp")
-        add_tests(name)
-        add_tests(name .. "_args", {runargs = {"foo", "bar"}})
-        add_tests(name .. "_pass_output", {runargs = "foo", pass_outputs = "hello foo"})
-        add_tests(name .. "_fail_output", {fail_outputs = {"hello .*", "hello xmake"}})
+        add_tests("default")
+        add_tests("args", {runargs = {"foo", "bar"}})
+        add_tests("pass_output", {runargs = "foo", pass_outputs = "hello foo"})
+        add_tests("fail_output", {fail_outputs = {"hello .*", "hello xmake"}})
 end
 
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index fe54119cf..045c8f3b3 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -25,6 +25,7 @@ import("core.project.config")
 import("core.base.global")
 import("core.project.project")
 import("core.platform.platform")
+import("core.theme.theme")
 import("async.runjobs")
 import("private.action.run.runenvs")
 import("private.service.remote_build.action", {alias = "remote_build_action"})
@@ -123,11 +124,7 @@ function _add_target_pkgenvs(target, targets_added)
 end
 
 -- run the given test
-function _run_test(test)
-
-    -- this target has been disabled?
-    local target = test.target
-    test.target = nil
+function _run_test(target, test)
 
     -- enter the environments of the target packages
     local oldenvs = os.getenvs()
@@ -177,8 +174,12 @@ end
 -- run tests
 function _run_tests(tests)
     local ordertests = {}
+    local maxwidth = 0
     for name, testinfo in table.orderpairs(tests) do
         table.insert(ordertests, testinfo)
+        if #testinfo.name > maxwidth then
+            maxwidth = #testinfo.name
+        end
     end
     if #ordertests == 0 then
         print("nothing to test")
@@ -193,10 +194,20 @@ function _run_tests(tests)
     runjobs("run_tests", function (index)
         local testinfo = ordertests[index]
         if testinfo then
-            local passed = _run_test(testinfo)
+            local target = testinfo.target
+            testinfo.target = nil
+            local spent = os.mclock()
+            local passed = _run_test(target, testinfo)
+            spent = os.mclock() - spent
             if passed then
                 report.passed = report.passed + 1
             end
+            local status_color = passed and "${color.success}" or "${color.failure}"
+            local progress_format = status_color .. theme.get("text.build.progress_format") .. ":${clear} "
+            local progress = math.floor(index * 100 / #ordertests)
+            local padding = maxwidth - #testinfo.name
+            cprint(progress_format .. "%s%s .................................... " .. status_color .. "%s${clear} ${bright}%0.3fs",
+                progress, testinfo.name, (" "):rep(padding), passed and "passed" or "failed", spent)
         end
     end, {total = #ordertests,
           comax = jobs,
@@ -205,7 +216,8 @@ function _run_tests(tests)
     -- generate report
     spent = os.mclock() - spent
     local passed_rate = math.floor(report.passed * 100 / report.total)
-    cprint("${color.success}%3d%%${clear} tests passed, ${color.failure}%d${clear} tests failed out of ${bright}%d${clear}, spent ${bright}%0.3fs",
+    print("")
+    cprint("${color.success}%d%%${clear} tests passed, ${color.failure}%d${clear} tests failed out of ${bright}%d${clear}, spent ${bright}%0.3fs",
         passed_rate, report.total - report.passed, report.total, spent / 1000)
 end
 
@@ -234,6 +246,7 @@ function main()
     for _, target in ipairs(project.ordertargets()) do
         if target:is_binary() or target:script("run") then
             for _, name in ipairs(target:get("tests")) do
+                name = target:name() .. "/" .. name
                 local testinfo = {name = name, target = target}
                 local extra = target:extraconf("tests", name)
                 if extra then
diff --git a/xmake/actions/test/xmake.lua b/xmake/actions/test/xmake.lua
index 4a48f4621..74cede2ff 100644
--- a/xmake/actions/test/xmake.lua
+++ b/xmake/actions/test/xmake.lua
@@ -40,7 +40,8 @@ task("test")
             {nil, "tests",     "vs",  nil   , "The test names. It support pattern matching.",
                                               "e.g.",
                                               "    xmake test foo",
-                                              "    xmake test foo_*"                                                }
+                                              "    xmake test */foo",
+                                              "    xmake test targetname/*"                                         }
         }
     }
 
-- 
cgit v1.3.1


From f37804a3a471f560c88e8f9311325037aa4f7083 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 13:38:29 +0800
Subject: improve run for test

---
 tests/actions/test/src/test_7.cpp |  2 +-
 tests/actions/test/src/test_8.cpp |  3 +-
 xmake/actions/test/main.lua       | 68 ++++++++++++++++++++++++++++++++-------
 3 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/tests/actions/test/src/test_7.cpp b/tests/actions/test/src/test_7.cpp
index f454e99f3..dae685668 100644
--- a/tests/actions/test/src/test_7.cpp
+++ b/tests/actions/test/src/test_7.cpp
@@ -6,5 +6,5 @@ int main(int argc, char** argv)
 {
     char const* arg = argc > 1? argv[1] : "xmake";
     cout << "hello " << arg << endl;
-    return 0;
+    return -1;
 }
diff --git a/tests/actions/test/src/test_8.cpp b/tests/actions/test/src/test_8.cpp
index f454e99f3..8669ff781 100644
--- a/tests/actions/test/src/test_8.cpp
+++ b/tests/actions/test/src/test_8.cpp
@@ -4,7 +4,6 @@ using namespace std;
 
 int main(int argc, char** argv)
 {
-    char const* arg = argc > 1? argv[1] : "xmake";
-    cout << "hello " << arg << endl;
+    cout << "hello xmake" << endl;
     return 0;
 }
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 045c8f3b3..8e25bc12e 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -44,17 +44,33 @@ function _do_test_target(target, opt)
 
     -- run test
     local outdata
+    local errors
     local rundir = opt.rundir or target:rundir()
     local targetfile = path.absolute(target:targetfile())
     local runargs = table.wrap(opt.runargs or target:get("runargs"))
-    local ok = try {
-        function ()
-            outdata = os.iorunv(targetfile, runargs, {curdir = rundir, envs = envs})
-            return true
+    local outfile = os.tmpfile()
+    local errfile = os.tmpfile()
+    local ok, syserrors = os.execv(targetfile, runargs, {try = true, curdir = rundir, envs = envs, stdout = outfile, stderr = errfile})
+    local outdata = os.isfile(outfile) and io.readfile(outfile)
+    if ok ~= 0 then
+        local errdata = os.isfile(errfile) and io.readfile(errfile)
+        errors = errdata or errors
+        if not errors or #errors == 0 then
+            local cmd = targetfile
+            if #runargs > 0 then
+                cmd = cmd .. " " .. os.args(runargs)
+            end
+            if ok ~= nil then
+                errors = string.format("run %s failed, exit code: %d", cmd, ok)
+            else
+                errors = string.format("run %s failed, exit error: %s", cmd, syserrors and syserrors or "unknown reason")
+            end
         end
-    }
+    end
+    os.tryrm(outfile)
+    os.tryrm(errfile)
 
-    if ok then
+    if ok == 0 then
         local passed
         outdata = outdata or ""
         for _, pass_output in ipairs(opt.pass_outputs) do
@@ -74,11 +90,23 @@ function _do_test_target(target, opt)
             if opt.plain then
                 if fail_output == outdata then
                     passed = false
+                    if not errors then
+                        errors = string.format("matched failed output: ${color.failure}%s${clear}", fail_output)
+                        if option.get("diagnosis") then
+                            errors = errors .. "\nactual output: " .. outdata
+                        end
+                    end
                     break
                 end
             else
                 if outdata:match("^" .. fail_output .. "$") then
                     passed = false
+                    if not errors then
+                        errors = string.format("matched failed output: ${color.failure}%s${clear}", fail_output)
+                        if option.get("diagnosis") then
+                            errors = errors .. "\nactual output: " .. outdata
+                        end
+                    end
                     break
                 end
             end
@@ -86,8 +114,15 @@ function _do_test_target(target, opt)
         if passed == nil then
             passed = true
         end
-        return passed
+        if passed == false and not errors and opt.passed_outputs then
+            errors = string.format("not matched passed output: ${color.success}%s${clear}", table.concat(opt.passed_outputs, ", "))
+            if option.get("diagnosis") then
+                errors = errors .. "\nactual output: " .. outdata
+            end
+        end
+        return passed, errors
     end
+    return false, errors
 end
 
 -- test target
@@ -95,16 +130,17 @@ function _on_test_target(target, opt)
 
     -- build target with rules
     local passed
+    local errors
     local done = false
     for _, r in ipairs(target:orderules()) do
         local on_test = r:script("test")
         if on_test then
-            passed = on_test(target, opt)
+            passed, errors = on_test(target, opt)
             done = true
         end
     end
     if done then
-        return passed
+        return passed, errors
     end
 
     -- do test
@@ -156,19 +192,21 @@ function _run_test(target, test)
 
     -- run the target scripts
     local passed
+    local errors
     for i = 1, 5 do
         local script = scripts[i]
         if script ~= nil then
-            local ok = script(target, test)
+            local ok, errs = script(target, test)
             if i == 3 then
                 passed = ok
+                errors = errs
             end
         end
     end
 
     -- leave the environments of the target packages
     os.setenvs(oldenvs)
-    return passed
+    return passed, errors
 end
 
 -- run tests
@@ -197,17 +235,23 @@ function _run_tests(tests)
             local target = testinfo.target
             testinfo.target = nil
             local spent = os.mclock()
-            local passed = _run_test(target, testinfo)
+            local passed, errors = _run_test(target, testinfo)
             spent = os.mclock() - spent
             if passed then
                 report.passed = report.passed + 1
             end
             local status_color = passed and "${color.success}" or "${color.failure}"
             local progress_format = status_color .. theme.get("text.build.progress_format") .. ":${clear} "
+            if option.get("verbose") then
+                progress_format = progress_format .. "${dim}"
+            end
             local progress = math.floor(index * 100 / #ordertests)
             local padding = maxwidth - #testinfo.name
             cprint(progress_format .. "%s%s .................................... " .. status_color .. "%s${clear} ${bright}%0.3fs",
                 progress, testinfo.name, (" "):rep(padding), passed and "passed" or "failed", spent)
+            if not passed and errors and (option.get("verbose") or option.get("diagnosis")) then
+                cprint(errors)
+            end
         end
     end, {total = #ordertests,
           comax = jobs,
-- 
cgit v1.3.1


From cfa128fdc1f18909396f9ae9845e0d9f71377b32 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 13:44:56 +0800
Subject: fix extra info

---
 tests/actions/test/xmake.lua | 2 +-
 xmake/actions/test/main.lua  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index a60d964a2..8f4467293 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -9,6 +9,6 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
         add_tests("default")
         add_tests("args", {runargs = {"foo", "bar"}})
         add_tests("pass_output", {runargs = "foo", pass_outputs = "hello foo"})
-        add_tests("fail_output", {fail_outputs = {"hello .*", "hello xmake"}})
+        add_tests("fail_output", {fail_outputs = {"hello2 .*", "hello xmake"}})
 end
 
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 8e25bc12e..5fb967b5b 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -290,9 +290,9 @@ function main()
     for _, target in ipairs(project.ordertargets()) do
         if target:is_binary() or target:script("run") then
             for _, name in ipairs(target:get("tests")) do
-                name = target:name() .. "/" .. name
-                local testinfo = {name = name, target = target}
                 local extra = target:extraconf("tests", name)
+                local testname = target:name() .. "/" .. name
+                local testinfo = {name = testname, target = target}
                 if extra then
                     table.join2(testinfo, extra)
                 end
@@ -302,7 +302,7 @@ function main()
 
                 local group = testinfo.group
                 if (not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
-                    tests[name] = testinfo
+                    tests[testname] = testinfo
                 end
             end
         end
-- 
cgit v1.3.1


From f116670bc4f6fc620134e6489391b3e8bd87d3e4 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 14:08:10 +0800
Subject: improve verbose output

---
 tests/actions/test/xmake.lua |  2 +-
 xmake/actions/test/main.lua  | 55 +++++++++++++++++++++++++++++++-------------
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/tests/actions/test/xmake.lua b/tests/actions/test/xmake.lua
index 8f4467293..086734631 100644
--- a/tests/actions/test/xmake.lua
+++ b/tests/actions/test/xmake.lua
@@ -8,7 +8,7 @@ for _, file in ipairs(os.files("src/test_*.cpp")) do
         add_files("src/" .. name .. ".cpp")
         add_tests("default")
         add_tests("args", {runargs = {"foo", "bar"}})
-        add_tests("pass_output", {runargs = "foo", pass_outputs = "hello foo"})
+        add_tests("pass_output", {trim_output = true, runargs = "foo", pass_outputs = "hello foo"})
         add_tests("fail_output", {fail_outputs = {"hello2 .*", "hello xmake"}})
 end
 
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 5fb967b5b..a566a4bc6 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -51,7 +51,10 @@ function _do_test_target(target, opt)
     local outfile = os.tmpfile()
     local errfile = os.tmpfile()
     local ok, syserrors = os.execv(targetfile, runargs, {try = true, curdir = rundir, envs = envs, stdout = outfile, stderr = errfile})
-    local outdata = os.isfile(outfile) and io.readfile(outfile)
+    local outdata = os.isfile(outfile) and io.readfile(outfile) or ""
+    if opt.trim_output then
+        outdata = outdata:trim()
+    end
     if ok ~= 0 then
         local errdata = os.isfile(errfile) and io.readfile(errfile)
         errors = errdata or errors
@@ -72,8 +75,9 @@ function _do_test_target(target, opt)
 
     if ok == 0 then
         local passed
-        outdata = outdata or ""
-        for _, pass_output in ipairs(opt.pass_outputs) do
+        local pass_outputs = table.wrap(opt.pass_outputs)
+        local fail_outputs = table.wrap(opt.fail_outputs)
+        for _, pass_output in ipairs(pass_outputs) do
             if opt.plain then
                 if pass_output == outdata then
                     passed = true
@@ -86,38 +90,57 @@ function _do_test_target(target, opt)
                 end
             end
         end
-        for _, fail_output in ipairs(opt.fail_outputs) do
+        for _, fail_output in ipairs(fail_outputs) do
             if opt.plain then
                 if fail_output == outdata then
                     passed = false
-                    if not errors then
+                    if not errors or #errors == 0 then
                         errors = string.format("matched failed output: ${color.failure}%s${clear}", fail_output)
-                        if option.get("diagnosis") then
-                            errors = errors .. "\nactual output: " .. outdata
+                        local actual_output = outdata
+                        if not option.get("diagnosis") then
+                            actual_output = outdata:sub(1, 64)
+                            if #outdata > #actual_output then
+                                actual_output = actual_output .. "..."
+                            end
                         end
+                        errors = errors .. ", actual output: ${color.failure}" .. actual_output
                     end
                     break
                 end
             else
                 if outdata:match("^" .. fail_output .. "$") then
                     passed = false
-                    if not errors then
+                    if not errors or #errors == 0 then
                         errors = string.format("matched failed output: ${color.failure}%s${clear}", fail_output)
-                        if option.get("diagnosis") then
-                            errors = errors .. "\nactual output: " .. outdata
+                        local actual_output = outdata
+                        if not option.get("diagnosis") then
+                            actual_output = outdata:sub(1, 64)
+                            if #outdata > #actual_output then
+                                actual_output = actual_output .. "..."
+                            end
                         end
+                        errors = errors .. ", actual output: ${color.failure}" .. actual_output
                     end
                     break
                 end
             end
         end
         if passed == nil then
-            passed = true
-        end
-        if passed == false and not errors and opt.passed_outputs then
-            errors = string.format("not matched passed output: ${color.success}%s${clear}", table.concat(opt.passed_outputs, ", "))
-            if option.get("diagnosis") then
-                errors = errors .. "\nactual output: " .. outdata
+            if #pass_outputs == 0 then
+                passed = true
+            else
+                passed = false
+                if not errors or #errors == 0 then
+                    errors = string.format("not matched passed output: ${color.success}%s${clear}", table.concat(pass_outputs, ", "))
+                    local actual_output = outdata
+                    if not option.get("diagnosis") then
+                        actual_output = outdata:sub(1, 64)
+                        if #outdata > #actual_output then
+                            actual_output = actual_output .. "..."
+                        end
+                    end
+                    errors = errors .. ", actual output: ${color.failure}" .. actual_output
+                end
             end
         end
         return passed, errors
-- 
cgit v1.3.1


From e230939c931a7b8b96da0c21f6c2d11c7fde2226 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 14:11:39 +0800
Subject: improve output

---
 tests/actions/test/src/test_5.cpp | 2 +-
 xmake/actions/test/main.lua       | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/actions/test/src/test_5.cpp b/tests/actions/test/src/test_5.cpp
index f454e99f3..dad841287 100644
--- a/tests/actions/test/src/test_5.cpp
+++ b/tests/actions/test/src/test_5.cpp
@@ -5,6 +5,6 @@ using namespace std;
 int main(int argc, char** argv)
 {
     char const* arg = argc > 1? argv[1] : "xmake";
-    cout << "hello " << arg << endl;
+    cout << "hello2 " << arg << endl;
     return 0;
 }
diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index a566a4bc6..d9dcf1936 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -59,14 +59,10 @@ function _do_test_target(target, opt)
         local errdata = os.isfile(errfile) and io.readfile(errfile)
         errors = errdata or errors
         if not errors or #errors == 0 then
-            local cmd = targetfile
-            if #runargs > 0 then
-                cmd = cmd .. " " .. os.args(runargs)
-            end
             if ok ~= nil then
-                errors = string.format("run %s failed, exit code: %d", cmd, ok)
+                errors = string.format("run failed, exit code: %d", ok)
             else
-                errors = string.format("run %s failed, exit error: %s", cmd, syserrors and syserrors or "unknown reason")
+                errors = string.format("run failed, exit error: %s", syserrors and syserrors or "unknown reason")
             end
         end
     end
-- 
cgit v1.3.1


From 8433602d73a41375efe45283dc9e209d373c966c Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 14:38:25 +0800
Subject: remove all option

---
 xmake/actions/test/main.lua  | 2 +-
 xmake/actions/test/xmake.lua | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index d9dcf1936..36c818a76 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -320,7 +320,7 @@ function main()
                 end
 
                 local group = testinfo.group
-                if (not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then
+                if (not group_pattern) or (group_pattern and group and group:match(group_pattern)) then
                     tests[testname] = testinfo
                 end
             end
diff --git a/xmake/actions/test/xmake.lua b/xmake/actions/test/xmake.lua
index 74cede2ff..c892d1840 100644
--- a/xmake/actions/test/xmake.lua
+++ b/xmake/actions/test/xmake.lua
@@ -25,8 +25,7 @@ task("test")
         usage = "xmake test [options] [target] [arguments]",
         description = "Run the project tests.",
         options = {
-            {'a', "all",        "k",   nil  , "Run all targets."                                                    },
-            {'g', "group",      "kv",  nil  , "Run all targets of the given group. It support path pattern matching.",
+            {'g', "group",      "kv",  nil  , "Run all tests of the given group. It support path pattern matching.",
                                               "e.g.",
                                               "    xmake test -g test",
                                               "    xmake test -g test_*",
-- 
cgit v1.3.1


From 8e42028cd8f1b8f1b0650f336fe445133e613f61 Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 14:47:07 +0800
Subject: improve config

---
 xmake/actions/test/main.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xmake/actions/test/main.lua b/xmake/actions/test/main.lua
index 36c818a76..d39c80a18 100644
--- a/xmake/actions/test/main.lua
+++ b/xmake/actions/test/main.lua
@@ -295,7 +295,7 @@ function main()
     project.lock()
 
     -- load config first
-    config.load()
+    task.run("config", {}, {disable_dump = true})
 
     -- load targets
     project.load_targets()
-- 
cgit v1.3.1


From 2bfdd8a805baa9753ffed3dac771b6f3dce077bf Mon Sep 17 00:00:00 2001
From: ruki <waruqi@gmail.com>
Date: Sat, 7 Oct 2023 15:20:12 +0800
Subject: update changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6324e935f..00bd1580d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 * [#4250](https://github.com/xmake-io/xmake/pull/4250): Improve link mechanism and order
 * [#1438](https://github.com/xmake-io/xmake/issues/1438): Support code amalgamation
+* [#3381](https://github.com/xmake-io/xmake/issues/3381): Add `xmake test` support
 
 ## v2.8.3
 
@@ -1671,6 +1672,7 @@
 
 * [#4250](https://github.com/xmake-io/xmake/pull/4250): 支持链接顺序调整，链接组
 * [#1438](https://github.com/xmake-io/xmake/issues/1438): 支持代码 amalgamation
+* [#3381](https://github.com/xmake-io/xmake/issues/3381): 添加 `xmake test` 支持
 
 ## v2.8.3
 
-- 
cgit v1.3.1