improve benchmark tests

2 files changed, 110 insertions, 59 deletions

diff --git a/tests/benchmarks/build_targets/test.lua b/tests/benchmarks/build_targets/test.lua
index 0f5124bfa..8660dbe52 100644
--- a/tests/benchmarks/build_targets/test.lua
+++ b/tests/benchmarks/build_targets/test.lua
@@ -1,6 +1,18 @@
 import("lib.detect.find_tool")
 import("core.tool.toolchain")
 
+function run_test(func)
+    local dt = os.mclock()
+    local n = 2
+    local delta = 0
+    for i = 1, n do
+        local e = func()
+        delta = delta + e
+    end
+    dt = os.mclock() - dt - delta
+    return math.floor(dt / n)
+end
+
 function test_build(t)
 
     if xmake.is_embed() then
@@ -10,29 +22,33 @@ function test_build(t)
     local jobs = tostring(os.default_njob())
 
     -- xmake
-    os.tryrm("build")
-    os.tryrm(".xmake")
-    local xmake_dt = os.mclock()
-    os.runv("xmake", {"-j" .. jobs})
-    xmake_dt = os.mclock() - xmake_dt
+    local xmake_dt = run_test(function()
+        local dt = os.mclock()
+        os.tryrm("build")
+        os.tryrm(".xmake")
+        dt = os.mclock() - dt
+        os.runv("xmake", {"-j" .. jobs})
+        return dt
+    end)
     print("build targets/30: xmake: %d ms", xmake_dt)
 
     -- cmake
     local cmake = find_tool("cmake")
     if cmake then
-        os.tryrm("build")
-        os.mkdir("build")
-        local cmake_default_dt = os.mclock()
-        os.runv(cmake.program, {".."}, {curdir = "build"})
-        os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
-        cmake_default_dt = os.mclock() - cmake_default_dt
+        local cmake_default_dt = run_test(function()
+            local dt = os.mclock()
+            os.tryrm("build")
+            os.mkdir("build")
+            dt = os.mclock() - dt
+            os.runv(cmake.program, {".."}, {curdir = "build"})
+            os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
+            return dt
+        end)
         print("build targets/30: cmake/default: %d ms", cmake_default_dt)
         t:require((cmake_default_dt > xmake_dt) or (cmake_default_dt + 3000 > xmake_dt))
 
         local ninja = find_tool("ninja")
         if ninja then
-            os.tryrm("build")
-            os.mkdir("build")
             local configs = {}
             local envs
             if is_host("windows") then
@@ -44,22 +60,30 @@ function test_build(t)
                     envs = os.joinenvs(msvc:runenvs())
                 end
             end
-            local cmake_ninja_dt = os.mclock()
-            os.runv(cmake.program, table.join("..", "-G", "Ninja", configs), {curdir = "build", envs = envs})
-            os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build", envs = envs})
-            cmake_ninja_dt = os.mclock() - cmake_ninja_dt
+            local cmake_ninja_dt = run_test(function ()
+                local dt = os.mclock()
+                os.tryrm("build")
+                os.mkdir("build")
+                dt = os.mclock() - dt
+                os.runv(cmake.program, table.join("..", "-G", "Ninja", configs), {curdir = "build", envs = envs})
+                os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build", envs = envs})
+                return dt
+            end)
             print("build targets/30: cmake/ninja: %d ms", cmake_ninja_dt)
             t:require((cmake_ninja_dt > xmake_dt) or (cmake_ninja_dt + 3000 > xmake_dt))
         end
 
         local make = find_tool("make")
         if make and not is_subhost("windows") then
-            os.tryrm("build")
-            os.mkdir("build")
-            local cmake_makefile_dt = os.mclock()
-            os.runv(cmake.program, {"..", "-G", "Unix Makefiles"}, {curdir = "build"})
-            os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
-            cmake_makefile_dt = os.mclock() - cmake_makefile_dt
+            local cmake_makefile_dt = run_test(function()
+                local dt = os.mclock()
+                os.tryrm("build")
+                os.mkdir("build")
+                dt = os.mclock() - dt
+                os.runv(cmake.program, {"..", "-G", "Unix Makefiles"}, {curdir = "build"})
+                os.runv(cmake.program, {"--build", ".", "-j" .. jobs}, {curdir = "build"})
+                return dt
+            end)
             print("build targets/30: cmake/makefile: %d ms", cmake_makefile_dt)
             t:require((cmake_makefile_dt > xmake_dt) or (cmake_makefile_dt + 3000 > xmake_dt))
         end
@@ -68,18 +92,18 @@ function test_build(t)
     -- meson
     local meson = find_tool("meson")
     if meson then
-        os.tryrm("build")
-        local meson_setup_dt = os.mclock()
-        os.runv(meson.program, {"setup", "build"})
-        meson_setup_dt = os.mclock() - meson_setup_dt
-
-        -- ccache will cache object files globally, which may affect the results of the second run.
-        io.replace("build/build.ninja", "ccache", "")
-
-        local meson_build_dt = os.mclock()
-        os.runv(meson.program, {"compile", "-j", jobs, "-C", "build"})
-        meson_build_dt = os.mclock() - meson_build_dt
-        local meson_dt = meson_setup_dt + meson_build_dt
+        local meson_dt = run_test(function()
+            local dt1 = os.mclock()
+            os.tryrm("build")
+            dt1 = os.mclock() - dt1
+            os.runv(meson.program, {"setup", "build"})
+            -- ccache will cache object files globally, which may affect the results of the second run.
+            local dt2 = os.mclock()
+            io.replace("build/build.ninja", "ccache", "")
+            dt2 = os.mclock() - dt2
+            os.runv(meson.program, {"compile", "-j", jobs, "-C", "build"})
+            return dt1 + dt2
+        end)
         print("build targets/30: meson: %d ms", meson_dt)
         t:require((meson_dt > xmake_dt) or (meson_dt + 3000 > xmake_dt))
     end
diff --git a/tests/benchmarks/config_targets/test.lua b/tests/benchmarks/config_targets/test.lua
index d5eb8b227..8eacf2dc1 100644
--- a/tests/benchmarks/config_targets/test.lua
+++ b/tests/benchmarks/config_targets/test.lua
@@ -1,6 +1,18 @@
 import("lib.detect.find_tool")
 import("core.tool.toolchain")
 
+function run_test(func)
+    local dt = os.mclock()
+    local n = 2
+    local delta = 0
+    for i = 1, n do
+        local e = func()
+        delta = delta + e
+    end
+    dt = os.mclock() - dt - delta
+    return math.floor(dt / n)
+end
+
 function test_config(t)
 
     if xmake.is_embed() then
@@ -8,28 +20,32 @@ function test_config(t)
     end
 
     -- xmake
-    os.tryrm("build")
-    os.tryrm(".xmake")
-    local xmake_dt = os.mclock()
-    os.runv("xmake", {"config", "-c"})
-    xmake_dt = os.mclock() - xmake_dt
+    local xmake_dt = run_test(function ()
+        local dt = os.mclock()
+        os.tryrm("build")
+        os.tryrm(".xmake")
+        dt = os.mclock() - dt
+        os.runv("xmake", {"config", "-c"})
+        return dt
+    end)
     print("config targets/1k: xmake: %d ms", xmake_dt)
 
     -- cmake
     local cmake = find_tool("cmake")
     if cmake then
-        os.tryrm("build")
-        os.mkdir("build")
-        local cmake_default_dt = os.mclock()
-        os.runv(cmake.program, {".."}, {curdir = "build"})
-        cmake_default_dt = os.mclock() - cmake_default_dt
+        local cmake_default_dt = run_test(function()
+            local dt = os.mclock()
+            os.tryrm("build")
+            os.mkdir("build")
+            dt = os.mclock() - dt
+            os.runv(cmake.program, {".."}, {curdir = "build"})
+            return dt
+        end)
         print("config targets/1k: cmake/default: %d ms", cmake_default_dt)
         t:require((cmake_default_dt > xmake_dt) or (cmake_default_dt + 2000 > xmake_dt))
 
         local ninja = find_tool("ninja")
         if ninja then
-            os.tryrm("build")
-            os.mkdir("build")
             local configs = {}
             local envs
             if is_host("windows") then
@@ -41,19 +57,27 @@ function test_config(t)
                     envs = os.joinenvs(msvc:runenvs())
                 end
             end
-            local cmake_ninja_dt = os.mclock()
-            os.runv(cmake.program, table.join("..", "-G", "Ninja", configs), {curdir = "build", envs = envs})
-            cmake_ninja_dt = os.mclock() - cmake_ninja_dt
+            local cmake_ninja_dt = run_test(function()
+                local dt = os.mclock()
+                os.tryrm("build")
+                os.mkdir("build")
+                dt = os.mclock() - dt
+                os.runv(cmake.program, table.join("..", "-G", "Ninja", configs), {curdir = "build", envs = envs})
+                return dt
+            end)
             print("config targets/1k: cmake/ninja: %d ms", cmake_ninja_dt)
             t:require((cmake_ninja_dt > xmake_dt) or (cmake_ninja_dt + 2000 > xmake_dt))
         end
 
         if find_tool("make") and not is_subhost("windows") then
-            os.tryrm("build")
-            os.mkdir("build")
-            local cmake_makefile_dt = os.mclock()
-            os.runv(cmake.program, {"..", "-G", "Unix Makefiles"}, {curdir = "build"})
-            cmake_makefile_dt = os.mclock() - cmake_makefile_dt
+            local cmake_makefile_dt = run_test(function ()
+                local dt = os.mclock()
+                os.tryrm("build")
+                os.mkdir("build")
+                dt = os.mclock() - dt
+                os.runv(cmake.program, {"..", "-G", "Unix Makefiles"}, {curdir = "build"})
+                return dt
+            end)
             print("config targets/1k: cmake/makefile: %d ms", cmake_makefile_dt)
             t:require((cmake_makefile_dt > xmake_dt) or (cmake_makefile_dt + 2000 > xmake_dt))
         end
@@ -62,10 +86,13 @@ function test_config(t)
     -- meson
     local meson = find_tool("meson")
     if meson then
-        os.tryrm("build")
-        local meson_dt = os.mclock()
-        os.runv(meson.program, {"setup", "build"})
-        meson_dt = os.mclock() - meson_dt
+        local meson_dt = run_test(function()
+            local dt = os.mclock()
+            os.tryrm("build")
+            dt = os.mclock() - dt
+            os.runv(meson.program, {"setup", "build"})
+            return dt
+        end)
         print("config targets/1k: meson: %d ms", meson_dt)
         t:require((meson_dt > xmake_dt) or (meson_dt + 2000 > xmake_dt))
     end