From 3d00c9106b461dae5866dcfb6ddbeed2276c4f31 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 Mar 2025 00:49:06 +0800 Subject: add on_prepare stages --- xmake/core/project/rule.lua | 9 +++++++++ xmake/core/project/target.lua | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index 571790fc5..a483943d8 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -287,6 +287,9 @@ function rule.apis() , "rule.on_test" , "rule.on_load" , "rule.on_config" + , "rule.on_prepare" + , "rule.on_prepare_file" + , "rule.on_prepare_files" , "rule.on_link" , "rule.on_build" , "rule.on_build_file" @@ -306,6 +309,9 @@ function rule.apis() , "rule.before_test" , "rule.before_load" , "rule.before_config" + , "rule.before_prepare" + , "rule.before_prepare_file" + , "rule.before_prepare_files" , "rule.before_link" , "rule.before_build" , "rule.before_build_file" @@ -325,6 +331,9 @@ function rule.apis() , "rule.after_test" , "rule.after_load" , "rule.after_config" + , "rule.after_prepare" + , "rule.after_prepare_file" + , "rule.after_prepare_files" , "rule.after_link" , "rule.after_build" , "rule.after_build_file" diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 360edb9cc..2dd779f15 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -2923,6 +2923,9 @@ function target.apis() , "target.on_test" , "target.on_load" , "target.on_config" + , "target.on_prepare" + , "target.on_prepare_file" + , "target.on_prepare_files" , "target.on_link" , "target.on_build" , "target.on_build_file" @@ -2936,6 +2939,10 @@ function target.apis() -- target.before_xxx , "target.before_run" , "target.before_test" + , "target.before_config" + , "target.before_prepare" + , "target.before_prepare_file" + , "target.before_prepare_files" , "target.before_link" , "target.before_build" , "target.before_build_file" @@ -2950,6 +2957,10 @@ function target.apis() , "target.after_run" , "target.after_test" , "target.after_load" + , "target.after_config" + , "target.after_prepare" + , "target.after_prepare_file" + , "target.after_prepare_files" , "target.after_link" , "target.after_build" , "target.after_build_file" -- cgit v1.3.1 From a1d0e9c838a21d14ff7e17f0cad3d6f3c3531ae3 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 Mar 2025 00:50:10 +0800 Subject: add missing target config --- xmake/core/sandbox/modules/import/core/project/project.lua | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/xmake/core/sandbox/modules/import/core/project/project.lua b/xmake/core/sandbox/modules/import/core/project/project.lua index 61b538bb1..c696b79b7 100644 --- a/xmake/core/sandbox/modules/import/core/project/project.lua +++ b/xmake/core/sandbox/modules/import/core/project/project.lua @@ -128,6 +128,10 @@ end -- config target function sandbox_core_project._config_target(target, opt) + local before_config = target:script("config_before") + if before_config then + before_config(target, opt) + end for _, rule in ipairs(table.wrap(target:orderules())) do local before_config = rule:script("config_before") if before_config then @@ -152,9 +156,13 @@ function sandbox_core_project._config_target(target, opt) after_config(target, opt) end end + local config_after = target:script("config_after") + if config_after then + config_after(target, opt) + end end --- config targets +-- config targets, TODO: We should support parallel configuration -- -- @param opt the extra option, e.g. {recheck = false} -- -- cgit v1.3.1 From 3a813e0ede0505a32ef9e4da7faeb792db0c40f3 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 13 Mar 2025 00:51:56 +0800 Subject: add prepare stub --- xmake/actions/build/build.lua | 5 ++++- xmake/actions/build/build_files.lua | 4 ++++ xmake/actions/build/prepare.lua | 26 ++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 xmake/actions/build/prepare.lua diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 28a33db78..8d5ffb72e 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -28,6 +28,7 @@ import("private.utils.batchcmds") import("core.base.hashset") import("private.service.remote_cache.client", {alias = "remote_cache_client"}) import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("prepare", {alias = "prepare_build"}) -- clean target for rebuilding function _clean_target(target) @@ -299,9 +300,11 @@ function get_batchjobs(targetnames, group_pattern) return batchjobs end --- the main entry function main(targetnames, group_pattern) + -- prepare to build + prepare_build(targetnames, {group_pattern = group_pattern}) + -- enable distcc? local distcc if distcc_build_client.is_connected() then diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index ad85d8b2a..c44b8ecd1 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -26,6 +26,7 @@ import("core.project.project") import("private.async.jobpool") import("async.runjobs") import("kinds.object") +import("prepare", {alias = "prepare_build"}) -- match source files function _match_sourcefiles(sourcefile, filepatterns) @@ -196,6 +197,9 @@ end -- the main entry function main(targetname, group_pattern, sourcefiles) + -- prepare to build + prepare_build(targetnames, {group_pattern = group_pattern, sourcefiles = sourcefiles}) + -- convert all sourcefiles to lua pattern local filepatterns = _get_file_patterns(sourcefiles) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua new file mode 100644 index 000000000..52a0152af --- /dev/null +++ b/xmake/actions/build/prepare.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare.lua +-- + +-- imports +import("core.base.option") +import("core.project.config") + +function main(targetnames, opt) +end -- cgit v1.3.1 From 00ebba39f7e81d2b52bfb3be458f9825e6d7c070 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 14 Mar 2025 00:53:08 +0800 Subject: add jobgraph stub --- tests/modules/async/jobgraph.lua | 7 ++++++ tests/modules/async/runjobs.lua | 43 +++++++++++++++++++++++++++++++++++++ tests/modules/scheduler/runjobs.lua | 43 ------------------------------------- xmake/modules/async/jobgraph.lua | 36 +++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 43 deletions(-) create mode 100644 tests/modules/async/jobgraph.lua create mode 100644 tests/modules/async/runjobs.lua delete mode 100644 tests/modules/scheduler/runjobs.lua create mode 100644 xmake/modules/async/jobgraph.lua diff --git a/tests/modules/async/jobgraph.lua b/tests/modules/async/jobgraph.lua new file mode 100644 index 000000000..dd8cb05f8 --- /dev/null +++ b/tests/modules/async/jobgraph.lua @@ -0,0 +1,7 @@ +import("core.base.scheduler") +import("async.jobgraph") + +function main() + +end + diff --git a/tests/modules/async/runjobs.lua b/tests/modules/async/runjobs.lua new file mode 100644 index 000000000..4e2a98cc0 --- /dev/null +++ b/tests/modules/async/runjobs.lua @@ -0,0 +1,43 @@ +import("core.base.scheduler") +import("private.async.jobpool") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + + -- test callback + print("==================================== test callback ====================================") + local t = os.mclock() + runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) + + -- test jobs + print("==================================== test jobs ====================================") + local jobs = jobpool.new() + local root = jobs:addjob("job/root", function (index, total, opt) + _jobfunc(index, total, opt) + end) + for i = 1, 3 do + local job = jobs:addjob("job/" .. i, function (index, total, opt) + _jobfunc(index, total, opt) + end, {rootjob = root}) + for j = 1, 50 do + jobs:addjob("job/" .. i .. "/" .. j, function (index, total, opt) + _jobfunc(index, total, opt) + end, {rootjob = job}) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/scheduler/runjobs.lua b/tests/modules/scheduler/runjobs.lua deleted file mode 100644 index 4e2a98cc0..000000000 --- a/tests/modules/scheduler/runjobs.lua +++ /dev/null @@ -1,43 +0,0 @@ -import("core.base.scheduler") -import("private.async.jobpool") -import("async.runjobs") - -function _jobfunc(index, total, opt) - print("%s: run job (%d/%d)", scheduler.co_running(), index, total) - local dt = os.mclock() - os.sleep(1000) - dt = os.mclock() - dt - print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) -end - -function main() - - -- test callback - print("==================================== test callback ====================================") - local t = os.mclock() - runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) - - -- test jobs - print("==================================== test jobs ====================================") - local jobs = jobpool.new() - local root = jobs:addjob("job/root", function (index, total, opt) - _jobfunc(index, total, opt) - end) - for i = 1, 3 do - local job = jobs:addjob("job/" .. i, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = root}) - for j = 1, 50 do - jobs:addjob("job/" .. i .. "/" .. j, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = job}) - end - end - t = os.mclock() - runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) -end - diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua new file mode 100644 index 000000000..519544cf2 --- /dev/null +++ b/xmake/modules/async/jobgraph.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file jobgraph.lua +-- + +-- imports +import("core.base.object") +import("core.base.graph") + +-- define module +local jobgraph = jobgraph or object {_init = {"_size"}} + +-- tostring +function jobgraph:__tostring() + return "" +end + +-- new a jobgraph +function new() + return jobgraph {0} +end -- cgit v1.3.1 From 1621db050a13474f4d79d02c4d306013770d35b4 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 18 Mar 2025 23:55:15 +0800 Subject: add jobgraph.new --- tests/modules/async/jobgraph.lua | 4 ++-- xmake/modules/async/jobgraph.lua | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/modules/async/jobgraph.lua b/tests/modules/async/jobgraph.lua index dd8cb05f8..7a0b87c65 100644 --- a/tests/modules/async/jobgraph.lua +++ b/tests/modules/async/jobgraph.lua @@ -1,7 +1,7 @@ -import("core.base.scheduler") import("async.jobgraph") function main() - + local gh = jobgraph.new() + print(gh) end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 519544cf2..3fd36194a 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -20,17 +20,23 @@ -- imports import("core.base.object") +import("core.base.list") import("core.base.graph") -- define module -local jobgraph = jobgraph or object {_init = {"_size"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_graph"}} + +-- get jobs +function jobgraph:jobs() + return self._jobs +end -- tostring function jobgraph:__tostring() - return "" + return string.format("", self:jobs():size()) end -- new a jobgraph function new() - return jobgraph {0} + return jobgraph {list.new(), graph.new(true)} end -- cgit v1.3.1 From 6785ce5822f298ce64684548dfc9d35ec8dd5030 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:03:25 +0800 Subject: improve runjobs tests --- tests/modules/async/jobgraph.lua | 7 ------ tests/modules/async/run_callback.lua | 19 ++++++++++++++++ tests/modules/async/run_jobgraph.lua | 28 +++++++++++++++++++++++ tests/modules/async/run_jobpool.lua | 28 +++++++++++++++++++++++ tests/modules/async/runjobs.lua | 43 ------------------------------------ xmake/modules/async/jobgraph.lua | 15 +++++++++++++ 6 files changed, 90 insertions(+), 50 deletions(-) delete mode 100644 tests/modules/async/jobgraph.lua create mode 100644 tests/modules/async/run_callback.lua create mode 100644 tests/modules/async/run_jobgraph.lua create mode 100644 tests/modules/async/run_jobpool.lua delete mode 100644 tests/modules/async/runjobs.lua diff --git a/tests/modules/async/jobgraph.lua b/tests/modules/async/jobgraph.lua deleted file mode 100644 index 7a0b87c65..000000000 --- a/tests/modules/async/jobgraph.lua +++ /dev/null @@ -1,7 +0,0 @@ -import("async.jobgraph") - -function main() - local gh = jobgraph.new() - print(gh) -end - diff --git a/tests/modules/async/run_callback.lua b/tests/modules/async/run_callback.lua new file mode 100644 index 000000000..21e75014b --- /dev/null +++ b/tests/modules/async/run_callback.lua @@ -0,0 +1,19 @@ +import("core.base.scheduler") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test callback ====================================") + local t = os.mclock() + runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua new file mode 100644 index 000000000..bcf5be2fd --- /dev/null +++ b/tests/modules/async/run_jobgraph.lua @@ -0,0 +1,28 @@ +import("core.base.scheduler") +import("async.jobgraph") +import("async.runjobs") + +function _jobfunc(job, opt) + print("%s: run job (%s)", scheduler.co_running(), job.name) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%s) end, progress: %s, dt: %d ms", scheduler.co_running(), job.name, opt.progress, dt) +end + +function main() + print("==================================== test jobpool ====================================") + local jobs = jobgraph.new() + jobs:add_job("job/root", _jobfunc) + for i = 1, 3 do + jobs:add_job("job/" .. i, _jobfunc) + for j = 1, 50 do + jobs:add_job("job/" .. i .. "/" .. j, _jobfunc) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/run_jobpool.lua b/tests/modules/async/run_jobpool.lua new file mode 100644 index 000000000..5de54e36c --- /dev/null +++ b/tests/modules/async/run_jobpool.lua @@ -0,0 +1,28 @@ +import("core.base.scheduler") +import("private.async.jobpool") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test jobpool ====================================") + local jobs = jobpool.new() + local root = jobs:addjob("job/root", _jobfunc) + for i = 1, 3 do + local job = jobs:addjob("job/" .. i, _jobfunc, {rootjob = root}) + for j = 1, 50 do + jobs:addjob("job/" .. i .. "/" .. j, _jobfunc, {rootjob = job}) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/runjobs.lua b/tests/modules/async/runjobs.lua deleted file mode 100644 index 4e2a98cc0..000000000 --- a/tests/modules/async/runjobs.lua +++ /dev/null @@ -1,43 +0,0 @@ -import("core.base.scheduler") -import("private.async.jobpool") -import("async.runjobs") - -function _jobfunc(index, total, opt) - print("%s: run job (%d/%d)", scheduler.co_running(), index, total) - local dt = os.mclock() - os.sleep(1000) - dt = os.mclock() - dt - print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) -end - -function main() - - -- test callback - print("==================================== test callback ====================================") - local t = os.mclock() - runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) - - -- test jobs - print("==================================== test jobs ====================================") - local jobs = jobpool.new() - local root = jobs:addjob("job/root", function (index, total, opt) - _jobfunc(index, total, opt) - end) - for i = 1, 3 do - local job = jobs:addjob("job/" .. i, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = root}) - for j = 1, 50 do - jobs:addjob("job/" .. i .. "/" .. j, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = job}) - end - end - t = os.mclock() - runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) -end - diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3fd36194a..30513ae56 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -31,6 +31,21 @@ function jobgraph:jobs() return self._jobs end +-- add a job to the jobgraph +-- +-- e.g. +-- jobgraph:add_job("xxx", function (job, opt) +-- end) +-- +-- @param name the job name +-- @param run the job run command/script +-- @param opt the job options +-- +function jobgraph:add_job(name, run, opt) + local job = {name = name, run = run, opt = opt} + self:jobs():insert(job) +end + -- tostring function jobgraph:__tostring() return string.format("", self:jobs():size()) -- cgit v1.3.1 From f8346ab6e40bdcf9506f1400f74860bbcea764ff Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:23:55 +0800 Subject: improve jobgraph tests --- tests/modules/async/run_jobgraph.lua | 13 ++++++----- xmake/modules/async/jobgraph.lua | 44 ++++++++++++++++++++++++++++++++---- xmake/modules/async/runjobs.lua | 5 ++++ 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index bcf5be2fd..ec9c9175f 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -2,22 +2,23 @@ import("core.base.scheduler") import("async.jobgraph") import("async.runjobs") -function _jobfunc(job, opt) - print("%s: run job (%s)", scheduler.co_running(), job.name) +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) local dt = os.mclock() os.sleep(1000) dt = os.mclock() - dt - print("%s: run job (%s) end, progress: %s, dt: %d ms", scheduler.co_running(), job.name, opt.progress, dt) + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) end function main() print("==================================== test jobpool ====================================") local jobs = jobgraph.new() - jobs:add_job("job/root", _jobfunc) + jobs:add("job/root", _jobfunc) for i = 1, 3 do - jobs:add_job("job/" .. i, _jobfunc) + jobs:add("job/" .. i, _jobfunc) for j = 1, 50 do - jobs:add_job("job/" .. i .. "/" .. j, _jobfunc) + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + jobs:add_deps("job/" .. i .. "/" .. j, "job/" .. i, "job/root") end end t = os.mclock() diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 30513ae56..313bc2d11 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -24,7 +24,16 @@ import("core.base.list") import("core.base.graph") -- define module -local jobgraph = jobgraph or object {_init = {"_jobs", "_graph"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_graph", "_dirty"}} + +-- remove the given job from the job queue +function jobqueue:remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() +end -- get jobs function jobgraph:jobs() @@ -34,24 +43,49 @@ end -- add a job to the jobgraph -- -- e.g. --- jobgraph:add_job("xxx", function (job, opt) +-- jobgraph:add("xxx", function (index, total, opt) -- end) -- -- @param name the job name -- @param run the job run command/script -- @param opt the job options -- -function jobgraph:add_job(name, run, opt) +function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} self:jobs():insert(job) + self._dirty = true +end + +-- remove a given job +function jobgraph:remove(name) + self._dirty = true +end + +-- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +function jobgraph:add_deps(...) + local deps = table.pack(...) +end + +-- add jog group +function jobgraph:add_group(name, callback) +end + +-- build a job queue +function jobgraph:build() + return jobqueue {self} +end + +-- get job size +function jobgraph:size() + return self:jobs():size() end -- tostring function jobgraph:__tostring() - return string.format("", self:jobs():size()) + return string.format("", self:size()) end -- new a jobgraph function new() - return jobgraph {list.new(), graph.new(true)} + return jobgraph {list.new(), graph.new(true), false} end diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index 0c2a6f366..959eb1e77 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -64,6 +64,11 @@ function main(name, jobs, opt) local group_name = name local jobs_cb = type(jobs) == "function" and jobs or nil assert(timeout < 60000, "runjobs: invalid timeout!") + + -- build jobs queue + if jobs.build then + jobs = jobs:build() + end assert(jobs, "runjobs: no jobs!") -- show waiting tips? -- cgit v1.3.1 From 08fade0d0b92ed766da0f752d375eda1126291f8 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:29:19 +0800 Subject: add build job queue stub --- xmake/modules/async/jobgraph.lua | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 313bc2d11..612e56dee 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,7 +25,23 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_jobs", "_graph", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_size", "_deps", "_dirty"}} + +-- build the job queue +function jobqueue:_build() + local graph = self._jobgraph + -- TODO + print("build job queue") +end + +-- update the job queue +function jobqueue:_update() + local graph = self._jobgraph + if graph._dirty then + self:_build() + graph._dirty = false + end +end -- remove the given job from the job queue function jobqueue:remove(job) @@ -33,11 +49,9 @@ end -- get a free job from the job queue function jobqueue:getfree() -end --- get jobs -function jobgraph:jobs() - return self._jobs + -- update the job queue first + self:_update() end -- add a job to the jobgraph @@ -51,23 +65,37 @@ end -- @param opt the job options -- function jobgraph:add(name, run, opt) - local job = {name = name, run = run, opt = opt} - self:jobs():insert(job) - self._dirty = true + local jobs = self._jobs + if not jobs[name] then + local job = {name = name, run = run, opt = opt} + jobs[name] = job + self._size = self._size + 1 + self._dirty = true + end end -- remove a given job function jobgraph:remove(name) - self._dirty = true + local jobs = self._jobs + if jobs[name] then + assert(self._size > 0) + jobs[name] = nil + self._size = self._size - 1 + self._dirty = true + end end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) + -- TODO local deps = table.pack(...) + self._dirty = true end -- add jog group function jobgraph:add_group(name, callback) + -- TODO + self._dirty = true end -- build a job queue @@ -75,9 +103,14 @@ function jobgraph:build() return jobqueue {self} end +-- get jobs +function jobgraph:jobs() + return self._jobs +end + -- get job size function jobgraph:size() - return self:jobs():size() + return self._size end -- tostring @@ -87,5 +120,5 @@ end -- new a jobgraph function new() - return jobgraph {list.new(), graph.new(true), false} + return jobgraph {{}, 0, graph.new(true), false} end -- cgit v1.3.1 From a588ee1c3f407cdfcbe5bc9753a5be1b7ca72e7e Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:30:01 +0800 Subject: fix runjobs --- xmake/modules/async/runjobs.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index 959eb1e77..d2af72843 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -66,7 +66,7 @@ function main(name, jobs, opt) assert(timeout < 60000, "runjobs: invalid timeout!") -- build jobs queue - if jobs.build then + if type(jobs) == "table" and jobs.build then jobs = jobs:build() end assert(jobs, "runjobs: no jobs!") -- cgit v1.3.1 From bc4f14679956d6eac336cb23d72e5e91c189a5bf Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:35:27 +0800 Subject: add deps for jobgraph --- tests/modules/async/run_jobgraph.lua | 2 +- xmake/modules/async/jobgraph.lua | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index ec9c9175f..a3e1edeb3 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -11,7 +11,7 @@ function _jobfunc(index, total, opt) end function main() - print("==================================== test jobpool ====================================") + print("==================================== test jobgraph ====================================") local jobs = jobgraph.new() jobs:add("job/root", _jobfunc) for i = 1, 3 do diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 612e56dee..684b7d7d8 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,7 +25,7 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_jobs", "_size", "_deps", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_deps", "_dirty"}} -- build the job queue function jobqueue:_build() @@ -87,9 +87,21 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) - -- TODO - local deps = table.pack(...) - self._dirty = true + local prev + local dirty + local jobs = self._jobs + local deps = self._deps + for _, name in ipairs(table.pack(...)) do + local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) + if prev then + deps:add_edge(prev, curr) + dirty = true + end + prev = curr + end + if dirty then + self._dirty = true + end end -- add jog group @@ -108,6 +120,11 @@ function jobgraph:jobs() return self._jobs end +-- get jobgraph name +function jobgraph:name() + return self._name +end + -- get job size function jobgraph:size() return self._size @@ -115,10 +132,10 @@ end -- tostring function jobgraph:__tostring() - return string.format("", self:size()) + return string.format("", self:name() or "anonymous", self:size()) end -- new a jobgraph -function new() - return jobgraph {{}, 0, graph.new(true), false} +function new(name) + return jobgraph {name, {}, 0, graph.new(true), false} end -- cgit v1.3.1 From 79522ada6117fb6bc87dc5cc7a0a88cfde5e12fc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 00:43:39 +0800 Subject: rename to dag --- xmake/modules/async/jobgraph.lua | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 684b7d7d8..ecd5ff7f1 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,13 +25,16 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_deps", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} -- build the job queue function jobqueue:_build() local graph = self._jobgraph - -- TODO - print("build job queue") + local dag = graph._dag + local queue = dag:topological_sort() + for _, v in ipairs(queue) do + print(v.name) + end end -- update the job queue @@ -89,12 +92,12 @@ end function jobgraph:add_deps(...) local prev local dirty + local dag = self._dag local jobs = self._jobs - local deps = self._deps for _, name in ipairs(table.pack(...)) do local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) if prev then - deps:add_edge(prev, curr) + dag:add_edge(prev, curr) dirty = true end prev = curr -- cgit v1.3.1 From 30ca6c18a4070e5f699f5fc61bf5fca475bcd67b Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 00:47:02 +0800 Subject: build job queue --- xmake/modules/async/jobgraph.lua | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ecd5ff7f1..458c20f5a 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -24,16 +24,17 @@ import("core.base.list") import("core.base.graph") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} -- build the job queue function jobqueue:_build() local graph = self._jobgraph local dag = graph._dag - local queue = dag:topological_sort() - for _, v in ipairs(queue) do - print(v.name) + local queue = self._queue + queue:clear() + for _, job in ipairs(dag:topological_sort()) do + queue:insert(job) end end @@ -48,13 +49,18 @@ end -- remove the given job from the job queue function jobqueue:remove(job) + local queue = self._queue + queue:remove(job) end -- get a free job from the job queue function jobqueue:getfree() - - -- update the job queue first self:_update() + + local queue = self._queue + if queue:empty() then + return + end end -- add a job to the jobgraph @@ -115,7 +121,7 @@ end -- build a job queue function jobgraph:build() - return jobqueue {self} + return jobqueue {self, list.new()} end -- get jobs -- cgit v1.3.1 From 2277ac87ba989b2b4004c07627235b48760579cc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:37:13 +0800 Subject: improve graph --- xmake/core/base/graph.lua | 50 +++++++++++++++++++++++++++------------- xmake/modules/async/jobgraph.lua | 13 ++++++++--- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 80d4566ec..1fb6b01dd 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -24,11 +24,11 @@ local object = require("base/object") -- define module local graph = graph or object { _init = {"_directed"} } {true} -local edge = edge or object { _init = {"_from", "_to", "_weight"} } +local edge = edge or object { _init = {"_from", "_to"} } -- new edge, from -> to -function edge.new(from, to, weight) - return edge {from, to, weight or 1.0} +function edge.new(from, to) + return edge {from, to} end function edge:from() @@ -47,8 +47,8 @@ function edge:other(v) end end -function edge:weight() - return self._weight +function edge:__tostring() + return string.format("", self:from(), self:to()) end -- clear graph @@ -56,6 +56,7 @@ function graph:clear() self._vertices = {} self._edges = {} self._adjacent_edges = {} + self._edges_map = {} end -- is empty? @@ -98,13 +99,19 @@ function graph:remove_vertex(v) end end) if contains then + self._edges_map[v] = nil self._adjacent_edges[v] = nil -- remove the adjacent edge with this vertex in the other vertices if not self:is_directed() then for _, w in ipairs(self:vertices()) do local edges = self:adjacent_edges(w) if edges then - table.remove_if(edges, function (_, e) return e:other(w) == v end) + table.remove_if(edges, function (_, e) + if e:other(w) == v then + self._edges_map[w] = nil + return true + end + end) end end end @@ -189,8 +196,8 @@ function graph:edges() end -- add edge -function graph:add_edge(from, to, weight) - local e = edge.new(from, to, weight) +function graph:add_edge(from, to) + local e = edge.new(from, to) if not self:has_vertex(from) then table.insert(self._vertices, from) self._adjacent_edges[from] = {} @@ -199,11 +206,16 @@ function graph:add_edge(from, to, weight) table.insert(self._vertices, to) self._adjacent_edges[to] = {} end + local edges_map = self._edges_map + edges_map[from] = edges_map[from] or {} + edges_map[from][to] = true if self:is_directed() then - table.insert(self._adjacent_edges[e:from()], e) + table.insert(self._adjacent_edges[from], e) else - table.insert(self._adjacent_edges[e:from()], e) - table.insert(self._adjacent_edges[e:to()], e) + table.insert(self._adjacent_edges[from], e) + table.insert(self._adjacent_edges[to], e) + edges_map[to] = edges_map[to] or {} + edges_map[to][from] = true end table.insert(self._edges, e) end @@ -212,9 +224,15 @@ end function graph:has_edge(from, to) local edges = self:adjacent_edges(from) if edges then - for _, e in ipairs(edges) do - if e:to() == to then - return true + local edges_map = self._edges_map + local from_map = edges_map[from] + if from_map and from_map[to] then + return true + else + for _, e in ipairs(edges) do + if e:to() == to then + return true + end end end end @@ -228,7 +246,7 @@ function graph:clone() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:from(), e:to(), e:weight()) + gh:add_edge(e:from(), e:to()) end end end @@ -245,7 +263,7 @@ function graph:reverse() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:to(), e:from(), e:weight()) + gh:add_edge(e:to(), e:from()) end end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 458c20f5a..bfeee3863 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -33,7 +33,9 @@ function jobqueue:_build() local dag = graph._dag local queue = self._queue queue:clear() + -- TODO find cycle for _, job in ipairs(dag:topological_sort()) do + print("build job", job.name) queue:insert(job) end end @@ -86,9 +88,12 @@ end -- remove a given job function jobgraph:remove(name) local jobs = self._jobs - if jobs[name] then + local job = jobs[name] + local dag = self._dag + if job then assert(self._size > 0) jobs[name] = nil + dag:remove_vertex(job) self._size = self._size - 1 self._dirty = true end @@ -103,8 +108,10 @@ function jobgraph:add_deps(...) for _, name in ipairs(table.pack(...)) do local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) if prev then - dag:add_edge(prev, curr) - dirty = true + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) + dirty = true + end end prev = curr end -- cgit v1.3.1 From 8954ac2acaf229a1007b37f4caadfb10a9d78fa4 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:39:52 +0800 Subject: check cirular dependencies --- xmake/modules/async/jobgraph.lua | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index bfeee3863..31a041911 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -32,8 +32,20 @@ function jobqueue:_build() local graph = self._jobgraph local dag = graph._dag local queue = self._queue + + -- check circular dependencies + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, job in ipairs(cycle) do + table.insert(names, job.name) + end + table.insert(names, names[1]) + raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) + end + + -- build job queue queue:clear() - -- TODO find cycle for _, job in ipairs(dag:topological_sort()) do print("build job", job.name) queue:insert(job) -- cgit v1.3.1 From 5b57870c2854f96c9fd2aebb6e9fe538b9a27b8c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:48:36 +0800 Subject: add job deps --- xmake/core/base/graph.lua | 9 +++++++-- xmake/modules/async/jobgraph.lua | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 1fb6b01dd..bf6b724ce 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -119,7 +119,8 @@ function graph:remove_vertex(v) end -- topological sort -function graph:topological_sort() +function graph:topological_sort(opt) + opt = opt or {} local visited = {} for _, v in ipairs(self:vertices()) do visited[v] = false @@ -143,7 +144,11 @@ function graph:topological_sort() dfs(v) end end - return table.reverse(order_vertices) + if opt.reverse then + return order_vertices + else + return table.reverse(order_vertices) + end end -- find cycle diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 31a041911..9bf98f7b5 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -22,11 +22,25 @@ import("core.base.object") import("core.base.list") import("core.base.graph") +import("core.base.hashset") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} +-- add job dependency +function jobqueue:_add_dep(job, dep) + job._deps = job._deps or hashset.new() + job._deps:insert(dep) + + local parents = dep._parents + if not parents then + parents = {} + dep._parents = parents + end + table.insert(parents, job) +end + -- build the job queue function jobqueue:_build() local graph = self._jobgraph @@ -46,10 +60,17 @@ function jobqueue:_build() -- build job queue queue:clear() - for _, job in ipairs(dag:topological_sort()) do - print("build job", job.name) + for _, job in ipairs(dag:topological_sort({reverse = true})) do + job._deps = nil + job._parents = nil queue:insert(job) end + + -- build job dependencies + for _, e in ipairs(dag:edges()) do + self:_add_dep(e:from(), e:to()) + print("%s -> %s", e:from().name, e:to().name) + end end -- update the job queue @@ -64,7 +85,9 @@ end -- remove the given job from the job queue function jobqueue:remove(job) local queue = self._queue + print("remove", job.name) queue:remove(job) + -- TODO remove deps end -- get a free job from the job queue @@ -75,6 +98,12 @@ function jobqueue:getfree() if queue:empty() then return end + + -- TODO + for job in queue:ritems() do + print("get free job", job.name) + return job + end end -- add a job to the jobgraph -- cgit v1.3.1 From a278d3bd3e74027e7fd1f878c7d8c991caf1a75f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 23:16:51 +0800 Subject: improve to find cycle --- tests/modules/graph/test.lua | 3 +++ xmake/core/base/graph.lua | 19 ++++++++++++---- xmake/core/tool/builder.lua | 11 ++++++---- xmake/modules/async/jobgraph.lua | 25 +++++++++++----------- .../modules/modules_support/dependency_scanner.lua | 25 +++++++++++----------- 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 63095be12..4bbefe069 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -52,5 +52,8 @@ function test_find_cycle(t) end local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) + + local _, has_cycle = dag:topological_sort() + t:require(has_cycle) end diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index bf6b724ce..3b88a642a 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -125,29 +125,40 @@ function graph:topological_sort(opt) for _, v in ipairs(self:vertices()) do visited[v] = false end + local in_stack = {} local order_vertices = {} local function dfs(v) visited[v] = true + in_stack[v] = true local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do local w = e:other(v) if not visited[w] then - dfs(w) + if dfs(w) then + return true + end + elseif in_stack[w] then + return true end end end + in_stack[v] = false table.insert(order_vertices, v) end + local has_cycle = false for _, v in ipairs(self:vertices()) do if not visited[v] then - dfs(v) + if dfs(v) then + has_cycle = true + break + end end end if opt.reverse then - return order_vertices + return order_vertices, has_cycle else - return table.reverse(order_vertices) + return table.reverse(order_vertices), has_cycle end end diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index 2bf4efc1f..b0f802237 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -668,11 +668,14 @@ function builder:_sort_links_of_items(items, opt) gh:add_edge(k, v) end if not gh:empty() then - local cycle = gh:find_cycle() - if cycle then - utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + local has_cycle + links, has_cycle = gh:topological_sort() + if has_cycle then + local cycle = gh:find_cycle() + if cycle then + utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + end end - links = gh:topological_sort() end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 9bf98f7b5..6d96ff0ff 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -47,20 +47,21 @@ function jobqueue:_build() local dag = graph._dag local queue = self._queue - -- check circular dependencies - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, job in ipairs(cycle) do - table.insert(names, job.name) - end - table.insert(names, names[1]) - raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) - end - -- build job queue queue:clear() - for _, job in ipairs(dag:topological_sort({reverse = true})) do + local order_jobs, has_cycle = dag:topological_sort({reverse = true}) + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, job in ipairs(cycle) do + table.insert(names, job.name) + end + table.insert(names, names[1]) + raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) + end + end + for _, job in ipairs(order_jobs) do job._deps = nil job._parents = nil queue:insert(job) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 9d552d0fd..df8c85441 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -222,7 +222,7 @@ function _generate_dependencies(target, sourcebatch, opt) local changed = false if opt.batchjobs then local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) + runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed end, {comax = jobs, total = #sourcebatch.sourcefiles}) @@ -415,19 +415,20 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, objectfile in ipairs(cycle) do - local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, objectfile in ipairs(cycle) do + local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + table.insert(names, name or cppfile) + end + local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) table.insert(names, name or cppfile) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) - table.insert(names, name or cppfile) - raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - - local objectfiles_sorted = table.reverse(dag:topological_sort()) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then @@ -465,7 +466,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) end end end - if insert then + if insert then table.insert(build_objectfiles, objectfile) table.insert(link_objectfiles, objectfile) elseif external and not external.from_moduleonly then -- cgit v1.3.1 From ce5fb205bbebb83fe6842b00858210f9d4256148 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 22:37:17 +0800 Subject: use kahn algorithm for graph by default --- xmake/core/base/graph.lua | 91 +++++++++++++++++++++++++++++++++++++--- xmake/modules/async/jobgraph.lua | 24 +---------- 2 files changed, 87 insertions(+), 28 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 3b88a642a..06324c0a0 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -118,9 +118,8 @@ function graph:remove_vertex(v) end end --- topological sort -function graph:topological_sort(opt) - opt = opt or {} +-- topological sort, use DFS algorithom +function graph:_topological_sort_dfs() local visited = {} for _, v in ipairs(self:vertices()) do visited[v] = false @@ -155,10 +154,90 @@ function graph:topological_sort(opt) end end end - if opt.reverse then - return order_vertices, has_cycle + return table.reverse(order_vertices), has_cycle +end + +-- topological sort, use Kahn's algorithm +function graph:_topological_sort_kahn() + + -- calculate in-degree for each vertex + local in_degree = {} + for _, v in ipairs(self:vertices()) do + in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + in_degree[w] = (in_degree[w] or 0) + 1 + end + end + end + end + + -- queue of vertices with no incoming edges (no dependencies) + local queue = {} + for _, v in ipairs(self:vertices()) do + if in_degree[v] == 0 then + table.insert(queue, v) + end + end + + -- result list for topologically sorted vertices + local order_vertices = {} + + -- process queue + while #queue > 0 do + -- remove a vertex with no incoming edges + local v = table.remove(queue, 1) + table.insert(order_vertices, v) + + -- for each outgoing edge, remove it and update in-degrees + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + in_degree[w] = in_degree[w] - 1 + -- if in-degree becomes zero, add to queue + if in_degree[w] == 0 then + table.insert(queue, w) + end + end + end + end + end + + -- if we couldn't process all vertices, there must be a cycle + local has_cycle = #order_vertices ~= #self:vertices() + + return order_vertices, has_cycle +end + +-- topological sort (default: Kahn's algorithm) +-- +-- @param opt the options, we can use `{algorithm = "dfs/kahn"}` to select sort algorithm, +-- and the Kahn is the default algorithm. +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- it will return {c, b, a} +function graph:topological_sort(opt) + opt = opt or {} + if not self:is_directed() then + return + end + if opt.algorithm == "dfs" then + return self:_topological_sort_dfs() else - return table.reverse(order_vertices), has_cycle + return self:_topological_sort_kahn() end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 6d96ff0ff..ca8725b28 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -28,19 +28,6 @@ import("core.base.hashset") local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} --- add job dependency -function jobqueue:_add_dep(job, dep) - job._deps = job._deps or hashset.new() - job._deps:insert(dep) - - local parents = dep._parents - if not parents then - parents = {} - dep._parents = parents - end - table.insert(parents, job) -end - -- build the job queue function jobqueue:_build() local graph = self._jobgraph @@ -49,7 +36,7 @@ function jobqueue:_build() -- build job queue queue:clear() - local order_jobs, has_cycle = dag:topological_sort({reverse = true}) + local order_jobs, has_cycle = dag:topological_sort() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -62,16 +49,9 @@ function jobqueue:_build() end end for _, job in ipairs(order_jobs) do - job._deps = nil - job._parents = nil + print("insert", job.name) queue:insert(job) end - - -- build job dependencies - for _, e in ipairs(dag:edges()) do - self:_add_dep(e:from(), e:to()) - print("%s -> %s", e:from().name, e:to().name) - end end -- update the job queue -- cgit v1.3.1 From 0662b04580e06e6d18b254e4005869569d8a8c50 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 22:38:32 +0800 Subject: use list as queue --- xmake/core/base/graph.lua | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 06324c0a0..681d390c3 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -19,7 +19,8 @@ -- -- load modules -local table = require("base/table") +local table = require("base/table") +local list = require("base/list") local object = require("base/object") -- define module @@ -180,10 +181,10 @@ function graph:_topological_sort_kahn() end -- queue of vertices with no incoming edges (no dependencies) - local queue = {} + local queue = list.new() for _, v in ipairs(self:vertices()) do if in_degree[v] == 0 then - table.insert(queue, v) + queue:insert(v) end end @@ -191,9 +192,9 @@ function graph:_topological_sort_kahn() local order_vertices = {} -- process queue - while #queue > 0 do + while not queue:empty() do -- remove a vertex with no incoming edges - local v = table.remove(queue, 1) + local v = queue:remove_first() table.insert(order_vertices, v) -- for each outgoing edge, remove it and update in-degrees @@ -205,7 +206,7 @@ function graph:_topological_sort_kahn() in_degree[w] = in_degree[w] - 1 -- if in-degree becomes zero, add to queue if in_degree[w] == 0 then - table.insert(queue, w) + queue:insert(w) end end end -- cgit v1.3.1 From 7e949212c4aff661f6dfd6234af7fd021a8568e3 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 22:41:46 +0800 Subject: remove dfs --- xmake/core/base/graph.lua | 75 ++++++++--------------------------------------- 1 file changed, 12 insertions(+), 63 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 681d390c3..ffcd815c5 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -119,47 +119,19 @@ function graph:remove_vertex(v) end end --- topological sort, use DFS algorithom -function graph:_topological_sort_dfs() - local visited = {} - for _, v in ipairs(self:vertices()) do - visited[v] = false - end - local in_stack = {} - local order_vertices = {} - local function dfs(v) - visited[v] = true - in_stack[v] = true - local edges = self:adjacent_edges(v) - if edges then - for _, e in ipairs(edges) do - local w = e:other(v) - if not visited[w] then - if dfs(w) then - return true - end - elseif in_stack[w] then - return true - end - end - end - in_stack[v] = false - table.insert(order_vertices, v) - end - local has_cycle = false - for _, v in ipairs(self:vertices()) do - if not visited[v] then - if dfs(v) then - has_cycle = true - break - end - end - end - return table.reverse(order_vertices), has_cycle -end - -- topological sort, use Kahn's algorithm -function graph:_topological_sort_kahn() +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- it will return {c, b, a} +function graph:topological_sort(opt) + opt = opt or {} + if not self:is_directed() then + return + end -- calculate in-degree for each vertex local in_degree = {} @@ -219,29 +191,6 @@ function graph:_topological_sort_kahn() return order_vertices, has_cycle end --- topological sort (default: Kahn's algorithm) --- --- @param opt the options, we can use `{algorithm = "dfs/kahn"}` to select sort algorithm, --- and the Kahn is the default algorithm. --- --- e.g. --- --- add_edge(a, b) -- a depend on b --- add_edge(b, c) -- b depend on c --- --- it will return {c, b, a} -function graph:topological_sort(opt) - opt = opt or {} - if not self:is_directed() then - return - end - if opt.algorithm == "dfs" then - return self:_topological_sort_dfs() - else - return self:_topological_sort_kahn() - end -end - -- find cycle function graph:find_cycle() local visited = {} -- cgit v1.3.1 From 741da62196bcb64c88e386f1a769462211c63b4e Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:03:29 +0800 Subject: add queue and improve graph --- tests/modules/graph/test.lua | 8 +- tests/modules/queue/test.lua | 35 ++++ xmake/core/base/graph.lua | 216 +++++++++++++++++++-- xmake/core/base/queue.lua | 125 ++++++++++++ .../sandbox/modules/import/core/base/queue.lua | 22 +++ xmake/core/tool/builder.lua | 2 +- xmake/modules/async/jobgraph.lua | 54 ++---- xmake/modules/cli/amalgamate.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 3 +- 9 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 tests/modules/queue/test.lua create mode 100644 xmake/core/base/queue.lua create mode 100644 xmake/core/sandbox/modules/import/core/base/queue.lua diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 4bbefe069..33e1f3fa7 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -1,6 +1,6 @@ import("core.base.graph") -function test_topological_sort(t) +function test_topo_sort(t) local edges = { {0, 5}, {0, 2}, @@ -18,7 +18,7 @@ function test_topological_sort(t) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local order_path = dag:topological_sort() + local order_path = dag:topo_sort() local orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -28,7 +28,7 @@ function test_topological_sort(t) end dag = dag:reverse() - order_path = dag:topological_sort() + order_path = dag:topo_sort() orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -53,7 +53,7 @@ function test_find_cycle(t) local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) - local _, has_cycle = dag:topological_sort() + local _, has_cycle = dag:topo_sort() t:require(has_cycle) end diff --git a/tests/modules/queue/test.lua b/tests/modules/queue/test.lua new file mode 100644 index 000000000..b577e04c9 --- /dev/null +++ b/tests/modules/queue/test.lua @@ -0,0 +1,35 @@ +import("core.base.queue") + +function test_push(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + t:are_equal(d:first(), 1) + t:are_equal(d:last(), 5) + local idx = 1 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + +function test_pop(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + d:pop() + t:are_equal(d:first(), 2) + t:are_equal(d:last(), 5) + local idx = 2 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index ffcd815c5..448eebb88 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -19,9 +19,10 @@ -- -- load modules -local table = require("base/table") -local list = require("base/list") -local object = require("base/object") +local table = require("base/table") +local queue = require("base/queue") +local object = require("base/object") +local hashset = require("base/hashset") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -58,6 +59,9 @@ function graph:clear() self._edges = {} self._adjacent_edges = {} self._edges_map = {} + + -- clear partial topological sort state + self:partial_topo_sort_reset() end -- is empty? @@ -116,10 +120,171 @@ function graph:remove_vertex(v) end end end + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() + end +end + +-- check if there's a cycle in the remaining unprocessed nodes +function graph:_check_cycle_in_remaining() + -- if all remaining nodes have in-degree > 0, we have a cycle + if self._topo_remaining_count > 0 and self._topo_remaining_count == self._topo_non_zero_indegree_count then + self._topo_has_cycle = true + return true end + return false end --- topological sort, use Kahn's algorithm +-- reset partial topological sort state +function graph:partial_topo_sort_reset() + self._topo_in_progress = false + self._topo_in_degree = nil + self._topo_queue = nil + self._topo_processed = nil + self._topo_has_cycle = nil + self._topo_remaining_count = nil + self._topo_non_zero_indegree_count = nil +end + +-- get next batch of nodes in topological order with limit +-- +-- @param limit the maximum number of nodes to return +-- @return array of nodes with zero in-degree, empty when complete +-- @return has_cycle indicates if a cycle was detected +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- local batch1, has_cycle = g:partial_topo_sort_next(1) -- returns {c} +-- local batch2, has_cycle = g:partial_topo_sort_next(1) -- returns {b} +-- local batch3, has_cycle = g:partial_topo_sort_next(1) -- returns {a} +-- local batch4, has_cycle = g:partial_topo_sort_next(1) -- returns {} (empty, all done) +-- +function graph:partial_topo_sort_next(limit) + if not self:is_directed() then + return {}, false + end + + limit = limit or math.huge + + -- check if we already detected a cycle + if self._topo_has_cycle then + return {}, true + end + + -- initialize topological sort state if not already in progress + if not self._topo_in_progress then + -- calculate in-degree for each vertex + self._topo_in_degree = {} + for _, v in ipairs(self:vertices()) do + self._topo_in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = (self._topo_in_degree[w] or 0) + 1 + end + end + end + end + + -- initialize queue with vertices that have no incoming edges + self._topo_queue = queue.new() + for _, v in ipairs(self:vertices()) do + if self._topo_in_degree[v] == 0 then + self._topo_queue:push(v) + end + end + + -- track processed vertices + self._topo_processed = hashset.new() + self._topo_in_progress = true + + -- track counts for efficient cycle detection + self._topo_remaining_count = #self:vertices() + self._topo_non_zero_indegree_count = self._topo_remaining_count - self._topo_queue:size() + + -- quick cycle detection: if no nodes have zero in-degree, we have a cycle + if self._topo_queue:empty() and self._topo_remaining_count > 0 then + self._topo_has_cycle = true + return {}, true + end + end + + -- return empty batch if queue is empty (all processed or cycle detected) + if self._topo_queue:empty() then + -- check if all vertices were processed + local processed_count = self._topo_processed:size() + self._topo_has_cycle = processed_count ~= #self:vertices() + + -- if this is the first call and we detect a cycle, mark as complete + if processed_count == 0 then + self._topo_in_progress = false + end + + return {}, self._topo_has_cycle + end + + -- collect up to 'limit' nodes with zero in-degree + local batch = {} + while not self._topo_queue:empty() and #batch < limit do + local v = self._topo_queue:pop() + table.insert(batch, v) + self._topo_processed:insert(v) + self._topo_remaining_count = self._topo_remaining_count - 1 + end + + -- update in-degrees based on the nodes in this batch + for _, v in ipairs(batch) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = self._topo_in_degree[w] - 1 + + -- update non-zero in-degree count + if self._topo_in_degree[w] == 0 then + self._topo_non_zero_indegree_count = self._topo_non_zero_indegree_count - 1 + + -- if in-degree becomes zero, add to queue for next batch + if not self._topo_processed:has(w) then + self._topo_queue:push(w) + end + end + end + end + end + end + + -- early cycle detection - if all remaining nodes have in-degree > 0 + if self:_check_cycle_in_remaining() then + return batch, true + end + + -- if queue is now empty and all vertices processed, reset state + if self._topo_queue:empty() then + local processed_count = self._topo_processed:size() + if processed_count == #self:vertices() then + self._topo_in_progress = false + else + -- if queue is empty but we still have unprocessed nodes, we have a cycle + self._topo_has_cycle = true + end + end + + return batch, self._topo_has_cycle +end + +-- topological sort, use kahn's algorithm -- -- e.g. -- @@ -127,8 +292,35 @@ end -- add_edge(b, c) -- b depend on c -- -- it will return {c, b, a} -function graph:topological_sort(opt) - opt = opt or {} +--[[ +function graph:topo_sort() + if not self:is_directed() then + return + end + + -- reset partial sort state to ensure we start fresh + self:partial_topo_sort_reset() + + local order_vertices = {} + local batch_size = math.huge -- no limit, get all at once + + -- get all nodes in one go + local batch, has_cycle = self:partial_topo_sort_next(batch_size) + while #batch > 0 do + for _, v in ipairs(batch) do + table.insert(order_vertices, v) + end + batch, has_cycle = self:partial_topo_sort_next(batch_size) + + -- quick exit if cycle is detected + if has_cycle then + break + end + end + + return order_vertices, has_cycle +end]] +function graph:topo_sort() if not self:is_directed() then return end @@ -153,10 +345,10 @@ function graph:topological_sort(opt) end -- queue of vertices with no incoming edges (no dependencies) - local queue = list.new() + local queue = queue.new() for _, v in ipairs(self:vertices()) do if in_degree[v] == 0 then - queue:insert(v) + queue:push(v) end end @@ -166,7 +358,7 @@ function graph:topological_sort(opt) -- process queue while not queue:empty() do -- remove a vertex with no incoming edges - local v = queue:remove_first() + local v = queue:pop() table.insert(order_vertices, v) -- for each outgoing edge, remove it and update in-degrees @@ -178,7 +370,7 @@ function graph:topological_sort(opt) in_degree[w] = in_degree[w] - 1 -- if in-degree becomes zero, add to queue if in_degree[w] == 0 then - queue:insert(w) + queue:push(w) end end end @@ -263,6 +455,9 @@ function graph:add_edge(from, to) edges_map[to][from] = true end table.insert(self._edges, e) + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() end -- has the given edge? @@ -340,4 +535,3 @@ end -- return module: graph return graph - diff --git a/xmake/core/base/queue.lua b/xmake/core/base/queue.lua new file mode 100644 index 000000000..dbdc1d87a --- /dev/null +++ b/xmake/core/base/queue.lua @@ -0,0 +1,125 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- load modules +local object = require("base/object") + +-- define module +local queue = queue or object {_init = {"_first", "_last"}} {1, 0} + +-- clear queue +function queue:clear() + self._first = 1 + self._last = 0 +end + +-- push item to queue +function queue:push(item) + local last = self._last + 1 + self._last = last + self[last] = item +end + +-- pop item from queue +function queue:pop() + local first = self._first + if first > self._last then + return nil + end + + local value = self[first] + self[first] = nil + self._first = first + 1 + return value +end + +-- get queue size +function queue:size() + return self._last - self._first + 1 +end + +-- is queue empty? +function queue:empty() + return self._first > self._last +end + +-- peek the first item of queue +function queue:first() + if self._first > self._last then + return nil + end + return self[self._first] +end + +-- peek the last item of queue +function queue:last() + if self._first > self._last then + return nil + end + return self[self._last] +end + +-- iterator for all items (forward) +-- +-- e.g. +-- +-- for item in queue:items() do +-- print(item) +-- end +-- +function queue:items() + local index = self._first - 1 + local last = self._last + return function() + index = index + 1 + if index <= last then + return self[index] + end + end +end + +-- iterator for all items (reverse) +function queue:ritems() + local index = self._last + 1 + local first = self._first + return function() + index = index - 1 + if index >= first then + return self[index] + end + end +end + +-- clone queue +function queue:clone() + local q = queue.new() + for i = self._first, self._last do + q:push(self[i]) + end + return q +end + +-- new queue +function queue.new() + return queue() +end + +-- return module: queue +return queue diff --git a/xmake/core/sandbox/modules/import/core/base/queue.lua b/xmake/core/sandbox/modules/import/core/base/queue.lua new file mode 100644 index 000000000..aab0a8214 --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/base/queue.lua @@ -0,0 +1,22 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- return module +return require("base/queue") diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index b0f802237..0e85911eb 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -669,7 +669,7 @@ function builder:_sort_links_of_items(items, opt) end if not gh:empty() then local has_cycle - links, has_cycle = gh:topological_sort() + links, has_cycle = gh:topo_sort() if has_cycle then local cycle = gh:find_cycle() if cycle then diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ca8725b28..4a0dac110 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,18 +25,20 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} --- build the job queue -function jobqueue:_build() +-- remove the given job from the job queue +function jobqueue:remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() local graph = self._jobgraph local dag = graph._dag - local queue = self._queue -- build job queue - queue:clear() - local order_jobs, has_cycle = dag:topological_sort() + local order_jobs, has_cycle = dag:partial_topo_sort_next(1) if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -48,42 +50,8 @@ function jobqueue:_build() raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end end - for _, job in ipairs(order_jobs) do - print("insert", job.name) - queue:insert(job) - end -end - --- update the job queue -function jobqueue:_update() - local graph = self._jobgraph - if graph._dirty then - self:_build() - graph._dirty = false - end -end - --- remove the given job from the job queue -function jobqueue:remove(job) - local queue = self._queue - print("remove", job.name) - queue:remove(job) - -- TODO remove deps -end - --- get a free job from the job queue -function jobqueue:getfree() - self:_update() - - local queue = self._queue - if queue:empty() then - return - end - - -- TODO - for job in queue:ritems() do - print("get free job", job.name) - return job + if order_jobs then + return table.unwrap(order_jobs) end end @@ -150,7 +118,7 @@ end -- build a job queue function jobgraph:build() - return jobqueue {self, list.new()} + return jobqueue {self} end -- get jobs diff --git a/xmake/modules/cli/amalgamate.lua b/xmake/modules/cli/amalgamate.lua index 8c7ec6fd7..d0a5ba40c 100644 --- a/xmake/modules/cli/amalgamate.lua +++ b/xmake/modules/cli/amalgamate.lua @@ -96,7 +96,7 @@ function _generate_file(target, inputpaths, outputpath, uniqueid) _generate_include_graph(target, inputpaths, gh, {}) -- sort file paths and remove root path - local filepaths = gh:topological_sort() + local filepaths = gh:topo_sort() table.remove(filepaths, 1) -- generate amalgamate file diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index df8c85441..94c14e60d 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -415,7 +415,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + local objectfiles_sorted, has_cycle = dag:topo_sort() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -429,6 +429,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end end + objectfiles_sorted = table.reverse(objectfiles_sorted) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then -- cgit v1.3.1 From 43ea0c1fb1d9954e065f3dfdb7bd93493d5d9e38 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:05:26 +0800 Subject: add partial topo test --- tests/modules/graph/test.lua | 59 ++++++++++++++++++++ xmake/core/base/graph.lua | 125 +++++++++++++++++-------------------------- 2 files changed, 109 insertions(+), 75 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 33e1f3fa7..3d5a0aede 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -38,6 +38,65 @@ function test_topo_sort(t) end end +function test_paritail_topo_sort(t) + local function partiail_topo_sort(dag) + dag:partial_topo_sort_reset() + + local order_vertices = {} + local batch_size = math.huge + local batch, has_cycle = dag:partial_topo_sort_next(batch_size) + while #batch > 0 do + for _, v in ipairs(batch) do + table.insert(order_vertices, v) + end + batch, has_cycle = dag:partial_topo_sort_next(batch_size) + + if has_cycle then + break + end + end + + return order_vertices, has_cycle + end + + local edges = { + {0, 5}, + {0, 2}, + {0, 1}, + {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + {6, 4}, + {6, 0}, + {3, 2}, + {1, 4}, + } + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) + end + local order_path = partiail_topo_sort(dag) + local orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + for _, e in ipairs(edges) do + t:require(orders[e[1]] < orders[e[2]]) + end + + dag = dag:reverse() + order_path = partiail_topo_sort(dag) + orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + for _, e in ipairs(edges) do + t:require(orders[e[1]] > orders[e[2]]) + end +end + + function test_find_cycle(t) local edges = { {9, 1}, diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 448eebb88..496bffeba 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -122,15 +122,15 @@ function graph:remove_vertex(v) end -- reset partial topological sort state since graph structure changed - self:partial_topo_sort_reset() + self._partial_topo_dirty = true end end -- check if there's a cycle in the remaining unprocessed nodes function graph:_check_cycle_in_remaining() -- if all remaining nodes have in-degree > 0, we have a cycle - if self._topo_remaining_count > 0 and self._topo_remaining_count == self._topo_non_zero_indegree_count then - self._topo_has_cycle = true + if self._partial_topo_remaining_count > 0 and self._partial_topo_remaining_count == self._partial_topo_non_zero_indegree_count then + self._partial_topo_has_cycle = true return true end return false @@ -138,13 +138,14 @@ end -- reset partial topological sort state function graph:partial_topo_sort_reset() - self._topo_in_progress = false - self._topo_in_degree = nil - self._topo_queue = nil - self._topo_processed = nil - self._topo_has_cycle = nil - self._topo_remaining_count = nil - self._topo_non_zero_indegree_count = nil + self._partial_topo_in_progress = false + self._partial_topo_in_degree = nil + self._partial_topo_queue = nil + self._partial_topo_processed = nil + self._partial_topo_has_cycle = nil + self._partial_topo_remaining_count = nil + self._partial_topo_non_zero_indegree_count = nil + self._partial_topo_dirty = false end -- get next batch of nodes in topological order with limit @@ -168,19 +169,23 @@ function graph:partial_topo_sort_next(limit) return {}, false end + if self._partial_topo_dirty then + self:partial_topo_sort_reset() + end + limit = limit or math.huge -- check if we already detected a cycle - if self._topo_has_cycle then + if self._partial_topo_has_cycle then return {}, true end -- initialize topological sort state if not already in progress - if not self._topo_in_progress then + if not self._partial_topo_in_progress then -- calculate in-degree for each vertex - self._topo_in_degree = {} + self._partial_topo_in_degree = {} for _, v in ipairs(self:vertices()) do - self._topo_in_degree[v] = 0 + self._partial_topo_in_degree[v] = 0 end -- count incoming edges for each vertex @@ -190,56 +195,56 @@ function graph:partial_topo_sort_next(limit) for _, e in ipairs(edges) do if e:from() == v then local w = e:to() - self._topo_in_degree[w] = (self._topo_in_degree[w] or 0) + 1 + self._partial_topo_in_degree[w] = (self._partial_topo_in_degree[w] or 0) + 1 end end end end -- initialize queue with vertices that have no incoming edges - self._topo_queue = queue.new() + self._partial_topo_queue = queue.new() for _, v in ipairs(self:vertices()) do - if self._topo_in_degree[v] == 0 then - self._topo_queue:push(v) + if self._partial_topo_in_degree[v] == 0 then + self._partial_topo_queue:push(v) end end -- track processed vertices - self._topo_processed = hashset.new() - self._topo_in_progress = true + self._partial_topo_processed = hashset.new() + self._partial_topo_in_progress = true -- track counts for efficient cycle detection - self._topo_remaining_count = #self:vertices() - self._topo_non_zero_indegree_count = self._topo_remaining_count - self._topo_queue:size() + self._partial_topo_remaining_count = #self:vertices() + self._partial_topo_non_zero_indegree_count = self._partial_topo_remaining_count - self._partial_topo_queue:size() -- quick cycle detection: if no nodes have zero in-degree, we have a cycle - if self._topo_queue:empty() and self._topo_remaining_count > 0 then - self._topo_has_cycle = true + if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then + self._partial_topo_has_cycle = true return {}, true end end -- return empty batch if queue is empty (all processed or cycle detected) - if self._topo_queue:empty() then + if self._partial_topo_queue:empty() then -- check if all vertices were processed - local processed_count = self._topo_processed:size() - self._topo_has_cycle = processed_count ~= #self:vertices() + local processed_count = self._partial_topo_processed:size() + self._partial_topo_has_cycle = processed_count ~= #self:vertices() -- if this is the first call and we detect a cycle, mark as complete if processed_count == 0 then - self._topo_in_progress = false + self._partial_topo_in_progress = false end - return {}, self._topo_has_cycle + return {}, self._partial_topo_has_cycle end -- collect up to 'limit' nodes with zero in-degree local batch = {} - while not self._topo_queue:empty() and #batch < limit do - local v = self._topo_queue:pop() + while not self._partial_topo_queue:empty() and #batch < limit do + local v = self._partial_topo_queue:pop() table.insert(batch, v) - self._topo_processed:insert(v) - self._topo_remaining_count = self._topo_remaining_count - 1 + self._partial_topo_processed:insert(v) + self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 end -- update in-degrees based on the nodes in this batch @@ -249,15 +254,15 @@ function graph:partial_topo_sort_next(limit) for _, e in ipairs(edges) do if e:from() == v then local w = e:to() - self._topo_in_degree[w] = self._topo_in_degree[w] - 1 + self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 -- update non-zero in-degree count - if self._topo_in_degree[w] == 0 then - self._topo_non_zero_indegree_count = self._topo_non_zero_indegree_count - 1 + if self._partial_topo_in_degree[w] == 0 then + self._partial_topo_non_zero_indegree_count = self._partial_topo_non_zero_indegree_count - 1 -- if in-degree becomes zero, add to queue for next batch - if not self._topo_processed:has(w) then - self._topo_queue:push(w) + if not self._partial_topo_processed:has(w) then + self._partial_topo_queue:push(w) end end end @@ -271,17 +276,17 @@ function graph:partial_topo_sort_next(limit) end -- if queue is now empty and all vertices processed, reset state - if self._topo_queue:empty() then - local processed_count = self._topo_processed:size() + if self._partial_topo_queue:empty() then + local processed_count = self._partial_topo_processed:size() if processed_count == #self:vertices() then - self._topo_in_progress = false + self._partial_topo_in_progress = false else -- if queue is empty but we still have unprocessed nodes, we have a cycle - self._topo_has_cycle = true + self._partial_topo_has_cycle = true end end - return batch, self._topo_has_cycle + return batch, self._partial_topo_has_cycle end -- topological sort, use kahn's algorithm @@ -292,34 +297,6 @@ end -- add_edge(b, c) -- b depend on c -- -- it will return {c, b, a} ---[[ -function graph:topo_sort() - if not self:is_directed() then - return - end - - -- reset partial sort state to ensure we start fresh - self:partial_topo_sort_reset() - - local order_vertices = {} - local batch_size = math.huge -- no limit, get all at once - - -- get all nodes in one go - local batch, has_cycle = self:partial_topo_sort_next(batch_size) - while #batch > 0 do - for _, v in ipairs(batch) do - table.insert(order_vertices, v) - end - batch, has_cycle = self:partial_topo_sort_next(batch_size) - - -- quick exit if cycle is detected - if has_cycle then - break - end - end - - return order_vertices, has_cycle -end]] function graph:topo_sort() if not self:is_directed() then return @@ -352,10 +329,8 @@ function graph:topo_sort() end end - -- result list for topologically sorted vertices - local order_vertices = {} - -- process queue + local order_vertices = {} while not queue:empty() do -- remove a vertex with no incoming edges local v = queue:pop() @@ -457,7 +432,7 @@ function graph:add_edge(from, to) table.insert(self._edges, e) -- reset partial topological sort state since graph structure changed - self:partial_topo_sort_reset() + self._partial_topo_dirty = true end -- has the given edge? -- cgit v1.3.1 From 093894844d4d7e7e58a7a78d36f04cf59b95ff31 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:12:16 +0800 Subject: improve tests --- tests/modules/graph/test.lua | 6 ++---- xmake/core/base/graph.lua | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 3d5a0aede..da6218f14 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -43,14 +43,12 @@ function test_paritail_topo_sort(t) dag:partial_topo_sort_reset() local order_vertices = {} - local batch_size = math.huge - local batch, has_cycle = dag:partial_topo_sort_next(batch_size) + local batch, has_cycle = dag:partial_topo_sort_next() while #batch > 0 do for _, v in ipairs(batch) do table.insert(order_vertices, v) end - batch, has_cycle = dag:partial_topo_sort_next(batch_size) - + batch, has_cycle = dag:partial_topo_sort_next() if has_cycle then break end diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 496bffeba..31e7038d1 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -165,6 +165,7 @@ end -- local batch4, has_cycle = g:partial_topo_sort_next(1) -- returns {} (empty, all done) -- function graph:partial_topo_sort_next(limit) + limit = limit or math.huge if not self:is_directed() then return {}, false end @@ -173,8 +174,6 @@ function graph:partial_topo_sort_next(limit) self:partial_topo_sort_reset() end - limit = limit or math.huge - -- check if we already detected a cycle if self._partial_topo_has_cycle then return {}, true -- cgit v1.3.1 From 53d124455837e9afa2c78e41b88f0f973fdf83e7 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:16:34 +0800 Subject: get parital single node --- tests/modules/graph/test.lua | 12 +++++------- xmake/core/base/graph.lua | 41 +++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index da6218f14..c9584a69a 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -42,16 +42,14 @@ function test_paritail_topo_sort(t) local function partiail_topo_sort(dag) dag:partial_topo_sort_reset() + local node, has_cycle local order_vertices = {} - local batch, has_cycle = dag:partial_topo_sort_next() - while #batch > 0 do - for _, v in ipairs(batch) do - table.insert(order_vertices, v) - end - batch, has_cycle = dag:partial_topo_sort_next() - if has_cycle then + while true do + node, has_cycle = dag:partial_topo_sort_next() + if node == nil or has_cycle then break end + table.insert(order_vertices, node) end return order_vertices, has_cycle diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 31e7038d1..b2d22a3c4 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -148,7 +148,7 @@ function graph:partial_topo_sort_reset() self._partial_topo_dirty = false end --- get next batch of nodes in topological order with limit +-- get next node in topological order -- -- @param limit the maximum number of nodes to return -- @return array of nodes with zero in-degree, empty when complete @@ -159,15 +159,15 @@ end -- add_edge(a, b) -- a depend on b -- add_edge(b, c) -- b depend on c -- --- local batch1, has_cycle = g:partial_topo_sort_next(1) -- returns {c} --- local batch2, has_cycle = g:partial_topo_sort_next(1) -- returns {b} --- local batch3, has_cycle = g:partial_topo_sort_next(1) -- returns {a} --- local batch4, has_cycle = g:partial_topo_sort_next(1) -- returns {} (empty, all done) +-- local node1, has_cycle = g:partial_topo_sort_next() -- returns c +-- local node2, has_cycle = g:partial_topo_sort_next() -- returns b +-- local node3, has_cycle = g:partial_topo_sort_next() -- returns a +-- local node4, has_cycle = g:partial_topo_sort_next() -- returns nil (empty, all done) -- function graph:partial_topo_sort_next(limit) limit = limit or math.huge if not self:is_directed() then - return {}, false + return nil, false end if self._partial_topo_dirty then @@ -176,7 +176,7 @@ function graph:partial_topo_sort_next(limit) -- check if we already detected a cycle if self._partial_topo_has_cycle then - return {}, true + return nil, true end -- initialize topological sort state if not already in progress @@ -219,7 +219,7 @@ function graph:partial_topo_sort_next(limit) -- quick cycle detection: if no nodes have zero in-degree, we have a cycle if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then self._partial_topo_has_cycle = true - return {}, true + return nil, true end end @@ -234,24 +234,21 @@ function graph:partial_topo_sort_next(limit) self._partial_topo_in_progress = false end - return {}, self._partial_topo_has_cycle + return nil, self._partial_topo_has_cycle end - -- collect up to 'limit' nodes with zero in-degree - local batch = {} - while not self._partial_topo_queue:empty() and #batch < limit do - local v = self._partial_topo_queue:pop() - table.insert(batch, v) - self._partial_topo_processed:insert(v) + -- get one node with zero in-degree + local node + if not self._partial_topo_queue:empty() then + node = self._partial_topo_queue:pop() + self._partial_topo_processed:insert(node) self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 - end - -- update in-degrees based on the nodes in this batch - for _, v in ipairs(batch) do - local edges = self:adjacent_edges(v) + -- update in-degrees based on the nodes in this batch + local edges = self:adjacent_edges(node) if edges then for _, e in ipairs(edges) do - if e:from() == v then + if e:from() == node then local w = e:to() self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 @@ -271,7 +268,7 @@ function graph:partial_topo_sort_next(limit) -- early cycle detection - if all remaining nodes have in-degree > 0 if self:_check_cycle_in_remaining() then - return batch, true + return node, true end -- if queue is now empty and all vertices processed, reset state @@ -285,7 +282,7 @@ function graph:partial_topo_sort_next(limit) end end - return batch, self._partial_topo_has_cycle + return node, self._partial_topo_has_cycle end -- topological sort, use kahn's algorithm -- cgit v1.3.1 From 5d6d739f00f781f91d30ffe93811128b968a151f Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:20:56 +0800 Subject: improve jobgraph --- xmake/core/base/graph.lua | 28 ++++++++-------------------- xmake/modules/async/jobgraph.lua | 31 ++++++++++--------------------- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index b2d22a3c4..5d4ac1abb 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -223,28 +223,19 @@ function graph:partial_topo_sort_next(limit) end end - -- return empty batch if queue is empty (all processed or cycle detected) + local node if self._partial_topo_queue:empty() then - -- check if all vertices were processed + -- return empty node if queue is empty (all processed or cycle detected) local processed_count = self._partial_topo_processed:size() self._partial_topo_has_cycle = processed_count ~= #self:vertices() - - -- if this is the first call and we detect a cycle, mark as complete - if processed_count == 0 then - self._partial_topo_in_progress = false - end - return nil, self._partial_topo_has_cycle - end - - -- get one node with zero in-degree - local node - if not self._partial_topo_queue:empty() then + else + -- get one node with zero in-degree node = self._partial_topo_queue:pop() self._partial_topo_processed:insert(node) self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 - -- update in-degrees based on the nodes in this batch + -- update in-degrees based on the nodes in this node local edges = self:adjacent_edges(node) if edges then for _, e in ipairs(edges) do @@ -256,7 +247,7 @@ function graph:partial_topo_sort_next(limit) if self._partial_topo_in_degree[w] == 0 then self._partial_topo_non_zero_indegree_count = self._partial_topo_non_zero_indegree_count - 1 - -- if in-degree becomes zero, add to queue for next batch + -- if in-degree becomes zero, add to queue for next node if not self._partial_topo_processed:has(w) then self._partial_topo_queue:push(w) end @@ -271,13 +262,10 @@ function graph:partial_topo_sort_next(limit) return node, true end - -- if queue is now empty and all vertices processed, reset state + -- if queue is empty but we still have unprocessed nodes, we have a cycle if self._partial_topo_queue:empty() then local processed_count = self._partial_topo_processed:size() - if processed_count == #self:vertices() then - self._partial_topo_in_progress = false - else - -- if queue is empty but we still have unprocessed nodes, we have a cycle + if processed_count ~= #self:vertices() then self._partial_topo_has_cycle = true end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 4a0dac110..46b10381b 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,20 +25,17 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} +local jobqueue = jobqueue or object {_init = {"_dag"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} --- remove the given job from the job queue +-- nothing to do, we need not to remove it function jobqueue:remove(job) end -- get a free job from the job queue function jobqueue:getfree() - local graph = self._jobgraph - local dag = graph._dag - - -- build job queue - local order_jobs, has_cycle = dag:partial_topo_sort_next(1) + local dag = self._dag + local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -50,9 +47,7 @@ function jobqueue:getfree() raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end end - if order_jobs then - return table.unwrap(order_jobs) - end + return freejob end -- add a job to the jobgraph @@ -71,7 +66,6 @@ function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 - self._dirty = true end end @@ -85,14 +79,12 @@ function jobgraph:remove(name) jobs[name] = nil dag:remove_vertex(job) self._size = self._size - 1 - self._dirty = true end end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) local prev - local dirty local dag = self._dag local jobs = self._jobs for _, name in ipairs(table.pack(...)) do @@ -100,25 +92,22 @@ function jobgraph:add_deps(...) if prev then if not dag:has_edge(prev, curr) then dag:add_edge(prev, curr) - dirty = true end end prev = curr end - if dirty then - self._dirty = true - end end -- add jog group function jobgraph:add_group(name, callback) -- TODO - self._dirty = true end -- build a job queue function jobgraph:build() - return jobqueue {self} + local dag = self._dag + dag:partial_topo_sort_reset() + return jobqueue {dag} end -- get jobs @@ -143,5 +132,5 @@ end -- new a jobgraph function new(name) - return jobgraph {name, {}, 0, graph.new(true), false} + return jobgraph {name, {}, 0, graph.new(true)} end -- cgit v1.3.1 From 33a96371dac5d18e489db6b6a69a55a61b2de9d2 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:20:29 +0800 Subject: init partial topo sort --- xmake/core/base/graph.lua | 110 +++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 5d4ac1abb..bd6c8a3cc 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -126,16 +126,6 @@ function graph:remove_vertex(v) end end --- check if there's a cycle in the remaining unprocessed nodes -function graph:_check_cycle_in_remaining() - -- if all remaining nodes have in-degree > 0, we have a cycle - if self._partial_topo_remaining_count > 0 and self._partial_topo_remaining_count == self._partial_topo_non_zero_indegree_count then - self._partial_topo_has_cycle = true - return true - end - return false -end - -- reset partial topological sort state function graph:partial_topo_sort_reset() self._partial_topo_in_progress = false @@ -159,10 +149,10 @@ end -- add_edge(a, b) -- a depend on b -- add_edge(b, c) -- b depend on c -- --- local node1, has_cycle = g:partial_topo_sort_next() -- returns c --- local node2, has_cycle = g:partial_topo_sort_next() -- returns b --- local node3, has_cycle = g:partial_topo_sort_next() -- returns a --- local node4, has_cycle = g:partial_topo_sort_next() -- returns nil (empty, all done) +-- local node1, has_cycle = g:partial_topo_sort_next() -- return c +-- local node2, has_cycle = g:partial_topo_sort_next() -- return b +-- local node3, has_cycle = g:partial_topo_sort_next() -- return a +-- local node4, has_cycle = g:partial_topo_sort_next() -- return nil (empty, all done) -- function graph:partial_topo_sort_next(limit) limit = limit or math.huge @@ -181,44 +171,9 @@ function graph:partial_topo_sort_next(limit) -- initialize topological sort state if not already in progress if not self._partial_topo_in_progress then - -- calculate in-degree for each vertex - self._partial_topo_in_degree = {} - for _, v in ipairs(self:vertices()) do - self._partial_topo_in_degree[v] = 0 - end - - -- count incoming edges for each vertex - for _, v in ipairs(self:vertices()) do - local edges = self:adjacent_edges(v) - if edges then - for _, e in ipairs(edges) do - if e:from() == v then - local w = e:to() - self._partial_topo_in_degree[w] = (self._partial_topo_in_degree[w] or 0) + 1 - end - end - end - end - - -- initialize queue with vertices that have no incoming edges - self._partial_topo_queue = queue.new() - for _, v in ipairs(self:vertices()) do - if self._partial_topo_in_degree[v] == 0 then - self._partial_topo_queue:push(v) - end - end - - -- track processed vertices - self._partial_topo_processed = hashset.new() + self:_partial_topo_sort_init() self._partial_topo_in_progress = true - - -- track counts for efficient cycle detection - self._partial_topo_remaining_count = #self:vertices() - self._partial_topo_non_zero_indegree_count = self._partial_topo_remaining_count - self._partial_topo_queue:size() - - -- quick cycle detection: if no nodes have zero in-degree, we have a cycle - if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then - self._partial_topo_has_cycle = true + if self._partial_topo_has_cycle then return nil, true end end @@ -485,6 +440,59 @@ function graph:dump() end end +-- initialize topological sort state if not already in progress +function graph:_partial_topo_sort_init() + + -- calculate in-degree for each vertex + self._partial_topo_in_degree = {} + for _, v in ipairs(self:vertices()) do + self._partial_topo_in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._partial_topo_in_degree[w] = (self._partial_topo_in_degree[w] or 0) + 1 + end + end + end + end + + -- initialize queue with vertices that have no incoming edges + self._partial_topo_queue = queue.new() + for _, v in ipairs(self:vertices()) do + if self._partial_topo_in_degree[v] == 0 then + self._partial_topo_queue:push(v) + end + end + + -- track processed vertices + self._partial_topo_processed = hashset.new() + + -- track counts for efficient cycle detection + self._partial_topo_remaining_count = #self:vertices() + self._partial_topo_non_zero_indegree_count = self._partial_topo_remaining_count - self._partial_topo_queue:size() + + -- quick cycle detection: if no nodes have zero in-degree, we have a cycle + if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then + self._partial_topo_has_cycle = true + end +end + +-- check if there's a cycle in the remaining unprocessed nodes +function graph:_check_cycle_in_remaining() + -- if all remaining nodes have in-degree > 0, we have a cycle + if self._partial_topo_remaining_count > 0 and self._partial_topo_remaining_count == self._partial_topo_non_zero_indegree_count then + self._partial_topo_has_cycle = true + return true + end + return false +end + -- new graph function graph.new(directed) local gh = graph {directed} -- cgit v1.3.1 From f1a16a647593691302154af0ff592a9cdf9cda5e Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:36:14 +0800 Subject: fix remove node --- tests/modules/graph/test.lua | 3 ++ xmake/core/base/graph.lua | 83 ++++++++++++---------------------------- xmake/modules/async/jobgraph.lua | 12 +++--- 3 files changed, 34 insertions(+), 64 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index c9584a69a..1fb0bcb43 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -50,6 +50,9 @@ function test_paritail_topo_sort(t) break end table.insert(order_vertices, node) + if node then + dag:partial_topo_sort_remove(node) + end end return order_vertices, has_cycle diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index bd6c8a3cc..04b0c6573 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -132,9 +132,8 @@ function graph:partial_topo_sort_reset() self._partial_topo_in_degree = nil self._partial_topo_queue = nil self._partial_topo_processed = nil + self._partial_topo_pending = 0 self._partial_topo_has_cycle = nil - self._partial_topo_remaining_count = nil - self._partial_topo_non_zero_indegree_count = nil self._partial_topo_dirty = false end @@ -173,59 +172,44 @@ function graph:partial_topo_sort_next(limit) if not self._partial_topo_in_progress then self:_partial_topo_sort_init() self._partial_topo_in_progress = true - if self._partial_topo_has_cycle then - return nil, true - end end + -- get one node with zero in-degree local node - if self._partial_topo_queue:empty() then - -- return empty node if queue is empty (all processed or cycle detected) - local processed_count = self._partial_topo_processed:size() - self._partial_topo_has_cycle = processed_count ~= #self:vertices() - return nil, self._partial_topo_has_cycle - else - -- get one node with zero in-degree + if not self._partial_topo_queue:empty() then node = self._partial_topo_queue:pop() self._partial_topo_processed:insert(node) - self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 - - -- update in-degrees based on the nodes in this node - local edges = self:adjacent_edges(node) - if edges then - for _, e in ipairs(edges) do - if e:from() == node then - local w = e:to() - self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 + self._partial_topo_pending = self._partial_topo_pending + 1 + end - -- update non-zero in-degree count - if self._partial_topo_in_degree[w] == 0 then - self._partial_topo_non_zero_indegree_count = self._partial_topo_non_zero_indegree_count - 1 + return node, self._partial_topo_has_cycle +end - -- if in-degree becomes zero, add to queue for next node - if not self._partial_topo_processed:has(w) then - self._partial_topo_queue:push(w) - end +-- remove node and update in-degrees based on the nodes in this node +function graph:partial_topo_sort_remove(node) + if node == nil then + return + end + self._partial_topo_pending = self._partial_topo_pending - 1 + local edges = self:adjacent_edges(node) + if edges then + for _, e in ipairs(edges) do + if e:from() == node then + local w = e:to() + self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 + if self._partial_topo_in_degree[w] == 0 then + if not self._partial_topo_processed:has(w) then + self._partial_topo_queue:push(w) end end end end end - -- early cycle detection - if all remaining nodes have in-degree > 0 - if self:_check_cycle_in_remaining() then - return node, true - end - - -- if queue is empty but we still have unprocessed nodes, we have a cycle - if self._partial_topo_queue:empty() then + if self._partial_topo_queue:empty() and self._partial_topo_pending == 0 then local processed_count = self._partial_topo_processed:size() - if processed_count ~= #self:vertices() then - self._partial_topo_has_cycle = true - end + self._partial_topo_has_cycle = processed_count ~= #self:vertices() end - - return node, self._partial_topo_has_cycle end -- topological sort, use kahn's algorithm @@ -472,25 +456,6 @@ function graph:_partial_topo_sort_init() -- track processed vertices self._partial_topo_processed = hashset.new() - - -- track counts for efficient cycle detection - self._partial_topo_remaining_count = #self:vertices() - self._partial_topo_non_zero_indegree_count = self._partial_topo_remaining_count - self._partial_topo_queue:size() - - -- quick cycle detection: if no nodes have zero in-degree, we have a cycle - if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then - self._partial_topo_has_cycle = true - end -end - --- check if there's a cycle in the remaining unprocessed nodes -function graph:_check_cycle_in_remaining() - -- if all remaining nodes have in-degree > 0, we have a cycle - if self._partial_topo_remaining_count > 0 and self._partial_topo_remaining_count == self._partial_topo_non_zero_indegree_count then - self._partial_topo_has_cycle = true - return true - end - return false end -- new graph diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 46b10381b..9994ac1f3 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,11 +25,13 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_dag"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph", "_dag"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} --- nothing to do, we need not to remove it +-- remove the finished job function jobqueue:remove(job) + local dag = self._dag + dag:partial_topo_sort_remove(job) end -- get a free job from the job queue @@ -37,15 +39,15 @@ function jobqueue:getfree() local dag = self._dag local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then + local names = {} local cycle = dag:find_cycle() if cycle then - local names = {} for _, job in ipairs(cycle) do table.insert(names, job.name) end table.insert(names, names[1]) - raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end + raise("%s: circular job dependency detected!\n%s", self._jobgraph, table.concat(names, "\n -> ")) end return freejob end @@ -107,7 +109,7 @@ end function jobgraph:build() local dag = self._dag dag:partial_topo_sort_reset() - return jobqueue {dag} + return jobqueue {self, dag} end -- get jobs -- cgit v1.3.1 From d579a1cc9322e67edd360e1a1dbd573f6d0320fc Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:37:43 +0800 Subject: check pending count --- xmake/core/base/graph.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 04b0c6573..63d339dec 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -190,6 +190,7 @@ function graph:partial_topo_sort_remove(node) if node == nil then return end + assert(self._partial_topo_pending > 0) self._partial_topo_pending = self._partial_topo_pending - 1 local edges = self:adjacent_edges(node) if edges then -- cgit v1.3.1 From ed3c482c9364cd79564bc62b15c0695d698c2fa3 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:40:54 +0800 Subject: optimize partial topo --- xmake/core/base/graph.lua | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 63d339dec..0c1f989ec 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -22,7 +22,6 @@ local table = require("base/table") local queue = require("base/queue") local object = require("base/object") -local hashset = require("base/hashset") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -131,8 +130,8 @@ function graph:partial_topo_sort_reset() self._partial_topo_in_progress = false self._partial_topo_in_degree = nil self._partial_topo_queue = nil - self._partial_topo_processed = nil - self._partial_topo_pending = 0 + self._partial_topo_processed = 0 + self._partial_topo_finished = 0 self._partial_topo_has_cycle = nil self._partial_topo_dirty = false end @@ -178,8 +177,7 @@ function graph:partial_topo_sort_next(limit) local node if not self._partial_topo_queue:empty() then node = self._partial_topo_queue:pop() - self._partial_topo_processed:insert(node) - self._partial_topo_pending = self._partial_topo_pending + 1 + self._partial_topo_processed = self._partial_topo_processed + 1 end return node, self._partial_topo_has_cycle @@ -190,8 +188,7 @@ function graph:partial_topo_sort_remove(node) if node == nil then return end - assert(self._partial_topo_pending > 0) - self._partial_topo_pending = self._partial_topo_pending - 1 + self._partial_topo_finished = self._partial_topo_finished + 1 local edges = self:adjacent_edges(node) if edges then for _, e in ipairs(edges) do @@ -199,17 +196,14 @@ function graph:partial_topo_sort_remove(node) local w = e:to() self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 if self._partial_topo_in_degree[w] == 0 then - if not self._partial_topo_processed:has(w) then - self._partial_topo_queue:push(w) - end + self._partial_topo_queue:push(w) end end end end - if self._partial_topo_queue:empty() and self._partial_topo_pending == 0 then - local processed_count = self._partial_topo_processed:size() - self._partial_topo_has_cycle = processed_count ~= #self:vertices() + if self._partial_topo_queue:empty() and self._partial_topo_processed == self._partial_topo_finished then + self._partial_topo_has_cycle = self._partial_topo_finished ~= #self:vertices() end end @@ -454,9 +448,6 @@ function graph:_partial_topo_sort_init() self._partial_topo_queue:push(v) end end - - -- track processed vertices - self._partial_topo_processed = hashset.new() end -- new graph -- cgit v1.3.1 From 63ed1854ea964fbbe7bc9dbdd8e2c34068dff7de Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:42:06 +0800 Subject: optimize remove node --- xmake/core/base/graph.lua | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 0c1f989ec..090d41ade 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -152,11 +152,7 @@ end -- local node3, has_cycle = g:partial_topo_sort_next() -- return a -- local node4, has_cycle = g:partial_topo_sort_next() -- return nil (empty, all done) -- -function graph:partial_topo_sort_next(limit) - limit = limit or math.huge - if not self:is_directed() then - return nil, false - end +function graph:partial_topo_sort_next() if self._partial_topo_dirty then self:partial_topo_sort_reset() @@ -169,7 +165,9 @@ function graph:partial_topo_sort_next(limit) -- initialize topological sort state if not already in progress if not self._partial_topo_in_progress then - self:_partial_topo_sort_init() + if not self:_partial_topo_sort_init() then + return nil, false + end self._partial_topo_in_progress = true end @@ -191,11 +189,13 @@ function graph:partial_topo_sort_remove(node) self._partial_topo_finished = self._partial_topo_finished + 1 local edges = self:adjacent_edges(node) if edges then + local partial_topo_in_degree = self._partial_topo_in_degree for _, e in ipairs(edges) do if e:from() == node then local w = e:to() - self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 - if self._partial_topo_in_degree[w] == 0 then + local in_degree = partial_topo_in_degree[w] - 1 + partial_topo_in_degree[w] = in_degree + if in_degree == 0 then self._partial_topo_queue:push(w) end end @@ -260,9 +260,9 @@ function graph:topo_sort() for _, e in ipairs(edges) do if e:from() == v then local w = e:to() - in_degree[w] = in_degree[w] - 1 - -- if in-degree becomes zero, add to queue - if in_degree[w] == 0 then + local d = in_degree[w] - 1 + in_degree[w] = d + if d == 0 then queue:push(w) end end @@ -272,7 +272,6 @@ function graph:topo_sort() -- if we couldn't process all vertices, there must be a cycle local has_cycle = #order_vertices ~= #self:vertices() - return order_vertices, has_cycle end @@ -421,6 +420,9 @@ end -- initialize topological sort state if not already in progress function graph:_partial_topo_sort_init() + if not self:is_directed() then + return false + end -- calculate in-degree for each vertex self._partial_topo_in_degree = {} @@ -432,10 +434,11 @@ function graph:_partial_topo_sort_init() for _, v in ipairs(self:vertices()) do local edges = self:adjacent_edges(v) if edges then + local partial_topo_in_degree = self._partial_topo_in_degree for _, e in ipairs(edges) do if e:from() == v then local w = e:to() - self._partial_topo_in_degree[w] = (self._partial_topo_in_degree[w] or 0) + 1 + partial_topo_in_degree[w] = (partial_topo_in_degree[w] or 0) + 1 end end end @@ -448,6 +451,7 @@ function graph:_partial_topo_sort_init() self._partial_topo_queue:push(v) end end + return true end -- new graph -- cgit v1.3.1 From 4a8219cda19cf320330ab7488f53158624c8c393 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:43:40 +0800 Subject: update comments --- tests/modules/graph/test.lua | 10 ++++++---- xmake/core/base/graph.lua | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 1fb0bcb43..c242962db 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -46,12 +46,14 @@ function test_paritail_topo_sort(t) local order_vertices = {} while true do node, has_cycle = dag:partial_topo_sort_next() - if node == nil or has_cycle then - break - end - table.insert(order_vertices, node) if node then + table.insert(order_vertices, node) dag:partial_topo_sort_remove(node) + else + if has_cycle then + raise("has cycle!") + end + break end end diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 090d41ade..4be7a2b77 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -142,6 +142,25 @@ end -- @return array of nodes with zero in-degree, empty when complete -- @return has_cycle indicates if a cycle was detected -- +-- @code +-- dag:partial_topo_sort_reset() +-- +-- local node, has_cycle +-- local order_vertices = {} +-- while true do +-- node, has_cycle = dag:partial_topo_sort_next() +-- if node then +-- table.insert(order_vertices, node) +-- dag:partial_topo_sort_remove(node) +-- else +-- if has_cycle then +-- -- find cycle +-- end +-- break +-- end +-- end +-- @endcode +-- -- e.g. -- -- add_edge(a, b) -- a depend on b -- cgit v1.3.1 From 172dbb827ab0efb3c32ebcac823ce866c18b2dd8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:04:10 +0800 Subject: improve tests --- tests/modules/graph/test.lua | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index c242962db..a13737b6e 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -72,6 +72,7 @@ function test_paritail_topo_sort(t) {6, 0}, {3, 2}, {1, 4}, + {2, 9}, } local dag = graph.new(true) for _, e in ipairs(edges) do @@ -97,6 +98,62 @@ function test_paritail_topo_sort(t) end end +function test_paritail_topo_sort_dynamic(t) + local function partiail_topo_sort(dag) + dag:partial_topo_sort_reset() + + local node, has_cycle + local order_vertices = {} + local dynamic_adjust = false + while true do + node, has_cycle = dag:partial_topo_sort_next() + if node then + if not dynamic_adjust then + dag:add_edge(1, 4) + dag:add_edge(2, 9) + dynamic_adjust = true + end + table.insert(order_vertices, node) + dag:partial_topo_sort_remove(node) + else + if has_cycle then + raise("has cycle!") + end + break + end + end + + assert(#order_vertices == #dag:vertices(), "vertices count not matched, %d != %d", #order_vertices, #dag:vertices()) + return order_vertices, has_cycle + end + + local edges = { + {0, 5}, + {0, 2}, + {0, 1}, + {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + {6, 4}, + {6, 0}, + {3, 2}, + } + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) + end + local order_path = partiail_topo_sort(dag) + local orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + table.insert(edges, {1, 4}) + table.insert(edges, {2, 9}) + for _, e in ipairs(edges) do + t:require(orders[e[1]] < orders[e[2]]) + end +end function test_find_cycle(t) local edges = { -- cgit v1.3.1 From a260aa4d879b459aeaf3526816f1bc474485f508 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:23:09 +0800 Subject: add group --- xmake/modules/async/jobgraph.lua | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 9994ac1f3..ee3a6f274 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -26,7 +26,7 @@ import("core.base.hashset") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph", "_dag"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_groups"}} -- remove the finished job function jobqueue:remove(job) @@ -60,7 +60,7 @@ end -- -- @param name the job name -- @param run the job run command/script --- @param opt the job options +-- @param opt the job options, e.g. {group = "xxx"} -- function jobgraph:add(name, run, opt) local jobs = self._jobs @@ -68,6 +68,16 @@ function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 + + local group_name = opt.group + if group_name then + local groups = self._groups[group_name] + if not groups then + groups = {} + self._groups[group_name] = groups + end + table.insert(groups, job) + end end end @@ -98,11 +108,8 @@ function jobgraph:add_deps(...) end prev = curr end -end - --- add jog group -function jobgraph:add_group(name, callback) -- TODO + -- add groups jobs end -- build a job queue @@ -134,5 +141,5 @@ end -- new a jobgraph function new(name) - return jobgraph {name, {}, 0, graph.new(true)} + return jobgraph {name, {}, 0, graph.new(true), {}} end -- cgit v1.3.1 From d31fc3eb8ade62b0d80280f22329bb132ea95cfc Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 22:28:30 +0800 Subject: support for job group --- tests/modules/async/run_jobgraph.lua | 26 ++++++++++++-- xmake/modules/async/jobgraph.lua | 68 +++++++++++++++++++++++++++++------- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index a3e1edeb3..bc1b5500f 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -10,8 +10,8 @@ function _jobfunc(index, total, opt) print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) end -function main() - print("==================================== test jobgraph ====================================") +function _test_basic() + print("==================================== test basic ====================================") local jobs = jobgraph.new() jobs:add("job/root", _jobfunc) for i = 1, 3 do @@ -27,3 +27,25 @@ function main() end}) end +function _test_group() + print("==================================== test group ====================================") + local jobs = jobgraph.new() + jobs:add("job/root", _jobfunc) + for i = 1, 3 do + jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) + end + end + jobs:add_deps("foo", "bar", "job/root") + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + +function main() + _test_basic() + _test_group() +end + diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ee3a6f274..edf3cfe81 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -37,6 +37,7 @@ end -- get a free job from the job queue function jobqueue:getfree() local dag = self._dag +::continue:: local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then local names = {} @@ -49,6 +50,11 @@ function jobqueue:getfree() end raise("%s: circular job dependency detected!\n%s", self._jobgraph, table.concat(names, "\n -> ")) end + -- if it's a fake job, we need to skip it and continue to get the next job + if freejob and not freejob.run then + dag:partial_topo_sort_remove(freejob) + goto continue + end return freejob end @@ -60,23 +66,25 @@ end -- -- @param name the job name -- @param run the job run command/script --- @param opt the job options, e.g. {group = "xxx"} +-- @param opt the job options, e.g. {groups = {"xxx"}} -- function jobgraph:add(name, run, opt) + opt = opt or {} local jobs = self._jobs if not jobs[name] then local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 - local group_name = opt.group - if group_name then - local groups = self._groups[group_name] - if not groups then - groups = {} - self._groups[group_name] = groups + if opt.groups then + for _, group_name in ipairs(opt.groups) do + local groups = self._groups[group_name] + if not groups then + groups = {} + self._groups[group_name] = groups + end + table.insert(groups, job) end - table.insert(groups, job) end end end @@ -97,19 +105,53 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) local prev + local prev_is_group local dag = self._dag local jobs = self._jobs + local groups = self._groups for _, name in ipairs(table.pack(...)) do - local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) + local curr_is_group = false + local curr = jobs[name] + if not curr then + curr = groups[name] + curr_is_group = true + end + assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then - if not dag:has_edge(prev, curr) then - dag:add_edge(prev, curr) + if prev_is_group and curr_is_group then + -- we use a fake task as a node to bridge the two groups. + local fakejob = {} + for _, job in ipairs(prev) do + if not dag:has_edge(job, fakejob) then + dag:add_edge(job, fakejob) + end + end + for _, job in ipairs(curr) do + if not dag:has_edge(fakejob, job) then + dag:add_edge(fakejob, job) + end + end + elseif curr_is_group then + for _, job in ipairs(curr) do + if not dag:has_edge(prev, job) then + dag:add_edge(prev, job) + end + end + elseif prev_is_group then + for _, job in ipairs(prev) do + if not dag:has_edge(job, curr) then + dag:add_edge(job, curr) + end + end + else + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) + end end end prev = curr + prev_is_group = curr_is_group end - -- TODO - -- add groups jobs end -- build a job queue -- cgit v1.3.1 From 5904d091ccdc6bde911d7ece763bd5253329ac9d Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 22:32:49 +0800 Subject: update comment --- xmake/modules/async/jobgraph.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index edf3cfe81..cc4d0c743 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -119,7 +119,7 @@ function jobgraph:add_deps(...) assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then if prev_is_group and curr_is_group then - -- we use a fake task as a node to bridge the two groups. + -- we use a fake job as a node to bridge the two groups. local fakejob = {} for _, job in ipairs(prev) do if not dag:has_edge(job, fakejob) then -- cgit v1.3.1 From 25c2b873ac04a5ef3d13a077b2ce8314f6b128f7 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 23 Mar 2025 00:11:33 +0800 Subject: recompute nodes for dirty graph --- xmake/core/base/graph.lua | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 4be7a2b77..dcb274318 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -22,6 +22,7 @@ local table = require("base/table") local queue = require("base/queue") local object = require("base/object") +local hashset = require("base/hashset") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -130,7 +131,7 @@ function graph:partial_topo_sort_reset() self._partial_topo_in_progress = false self._partial_topo_in_degree = nil self._partial_topo_queue = nil - self._partial_topo_processed = 0 + self._partial_topo_processed = nil self._partial_topo_finished = 0 self._partial_topo_has_cycle = nil self._partial_topo_dirty = false @@ -173,8 +174,9 @@ end -- function graph:partial_topo_sort_next() + -- recompute all nodes if has dirty nodes if self._partial_topo_dirty then - self:partial_topo_sort_reset() + self:_partial_topo_sort_recompute_all() end -- check if we already detected a cycle @@ -192,9 +194,17 @@ function graph:partial_topo_sort_next() -- get one node with zero in-degree local node - if not self._partial_topo_queue:empty() then - node = self._partial_topo_queue:pop() - self._partial_topo_processed = self._partial_topo_processed + 1 + local partial_topo_queue = self._partial_topo_queue + local partial_topo_processed = self._partial_topo_processed + while not partial_topo_queue:empty() do + local v = partial_topo_queue:pop() + if partial_topo_processed:has(v) then + self:partial_topo_sort_remove(v) + else + node = v + partial_topo_processed:insert(node) + break + end end return node, self._partial_topo_has_cycle @@ -209,19 +219,20 @@ function graph:partial_topo_sort_remove(node) local edges = self:adjacent_edges(node) if edges then local partial_topo_in_degree = self._partial_topo_in_degree + local partial_topo_queue = self._partial_topo_queue for _, e in ipairs(edges) do if e:from() == node then local w = e:to() local in_degree = partial_topo_in_degree[w] - 1 partial_topo_in_degree[w] = in_degree if in_degree == 0 then - self._partial_topo_queue:push(w) + partial_topo_queue:push(w) end end end end - if self._partial_topo_queue:empty() and self._partial_topo_processed == self._partial_topo_finished then + if self._partial_topo_queue:empty() and self._partial_topo_processed:size() == self._partial_topo_finished then self._partial_topo_has_cycle = self._partial_topo_finished ~= #self:vertices() end end @@ -450,10 +461,10 @@ function graph:_partial_topo_sort_init() end -- count incoming edges for each vertex + local partial_topo_in_degree = self._partial_topo_in_degree for _, v in ipairs(self:vertices()) do local edges = self:adjacent_edges(v) if edges then - local partial_topo_in_degree = self._partial_topo_in_degree for _, e in ipairs(edges) do if e:from() == v then local w = e:to() @@ -465,14 +476,27 @@ function graph:_partial_topo_sort_init() -- initialize queue with vertices that have no incoming edges self._partial_topo_queue = queue.new() + local partial_topo_queue = self._partial_topo_queue for _, v in ipairs(self:vertices()) do - if self._partial_topo_in_degree[v] == 0 then - self._partial_topo_queue:push(v) + if partial_topo_in_degree[v] == 0 then + partial_topo_queue:push(v) end end + + self._partial_topo_processed = self._partial_topo_processed or hashset.new() return true end +-- recompute all nodes +function graph:_partial_topo_sort_recompute_all() + self._partial_topo_in_progress = false + self._partial_topo_in_degree = nil + self._partial_topo_queue = nil + self._partial_topo_finished = 0 + self._partial_topo_has_cycle = nil + self._partial_topo_dirty = false +end + -- new graph function graph.new(directed) local gh = graph {directed} -- cgit v1.3.1 From 9b3f6f86ae92c9fb779fbb898801ab4c70a6390e Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 23 Mar 2025 21:46:33 +0800 Subject: fix remove vertex --- tests/modules/graph/test.lua | 23 +++++++++++++++++++---- xmake/core/base/graph.lua | 28 ++++++++++++++-------------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index a13737b6e..fe1d63e8d 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -110,11 +110,14 @@ function test_paritail_topo_sort_dynamic(t) if node then if not dynamic_adjust then dag:add_edge(1, 4) - dag:add_edge(2, 9) - dynamic_adjust = true + dag:remove_vertex(6) end table.insert(order_vertices, node) dag:partial_topo_sort_remove(node) + if not dynamic_adjust then + dag:add_edge(2, 9) + dynamic_adjust = true + end else if has_cycle then raise("has cycle!") @@ -148,8 +151,20 @@ function test_paritail_topo_sort_dynamic(t) for i, v in ipairs(order_path) do orders[v] = i end - table.insert(edges, {1, 4}) - table.insert(edges, {2, 9}) + edges = { + {0, 5}, + {0, 2}, + {0, 1}, + -- {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + -- {6, 4}, + -- {6, 0}, + {3, 2}, + {1, 4}, + {2, 9} + } for _, e in ipairs(edges) do t:require(orders[e[1]] < orders[e[2]]) end diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index dcb274318..658b71e73 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -107,17 +107,15 @@ function graph:remove_vertex(v) self._edges_map[v] = nil self._adjacent_edges[v] = nil -- remove the adjacent edge with this vertex in the other vertices - if not self:is_directed() then - for _, w in ipairs(self:vertices()) do - local edges = self:adjacent_edges(w) - if edges then - table.remove_if(edges, function (_, e) - if e:other(w) == v then - self._edges_map[w] = nil - return true - end - end) - end + for _, w in ipairs(self:vertices()) do + local edges = self:adjacent_edges(w) + if edges then + table.remove_if(edges, function (_, e) + if e:other(w) == v then + self._edges_map[w] = nil + return true + end + end) end end @@ -176,7 +174,7 @@ function graph:partial_topo_sort_next() -- recompute all nodes if has dirty nodes if self._partial_topo_dirty then - self:_partial_topo_sort_recompute_all() + self:_partial_topo_sort_recompute_dirty() end -- check if we already detected a cycle @@ -487,8 +485,10 @@ function graph:_partial_topo_sort_init() return true end --- recompute all nodes -function graph:_partial_topo_sort_recompute_all() +-- recompute all dirty nodes +-- +-- TODO we recompute all nodes now, but we should optimize to recompute only dirty nodes +function graph:_partial_topo_sort_recompute_dirty() self._partial_topo_in_progress = false self._partial_topo_in_degree = nil self._partial_topo_queue = nil -- cgit v1.3.1 From b8b96a08063228fd7832862275c9f47c24b1546c Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 23 Mar 2025 23:33:09 +0800 Subject: prepare files --- xmake/actions/build/build.lua | 9 +++++---- xmake/actions/build/build_files.lua | 22 +++++++++++----------- xmake/actions/build/main.lua | 14 ++++++-------- xmake/actions/build/prepare_files.lua | 26 ++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 xmake/actions/build/prepare_files.lua diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 8d5ffb72e..d11b1841b 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -231,7 +231,7 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, end -- get batch jobs, @note we need to export it for private.diagnosis.dump_buildjobs -function get_batchjobs(targetnames, group_pattern) +function get_batchjobs(targetnames, opt) -- get root targets local targets_root = {} @@ -251,6 +251,7 @@ function get_batchjobs(targetnames, group_pattern) end end else + local group_pattern = opt.group_pattern local depset = hashset.new() local targets = {} for _, target in ipairs(project.ordertargets()) do @@ -300,10 +301,10 @@ function get_batchjobs(targetnames, group_pattern) return batchjobs end -function main(targetnames, group_pattern) +function main(targetnames, opt) -- prepare to build - prepare_build(targetnames, {group_pattern = group_pattern}) + prepare_build(targetnames, opt) -- enable distcc? local distcc @@ -312,7 +313,7 @@ function main(targetnames, group_pattern) end -- build all jobs - local batchjobs = get_batchjobs(targetnames, group_pattern) + local batchjobs = get_batchjobs(targetnames, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build", batchjobs, {on_exit = function (errors) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index c44b8ecd1..8848962db 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -26,7 +26,7 @@ import("core.project.project") import("private.async.jobpool") import("async.runjobs") import("kinds.object") -import("prepare", {alias = "prepare_build"}) +import("prepare_files", {alias = "prepare_build_files"}) -- match source files function _match_sourcefiles(sourcefile, filepatterns) @@ -49,7 +49,6 @@ end -- add batch jobs function _add_batchjobs(batchjobs, rootjob, target, filepatterns) - local newbatches = {} local sourcecount = 0 for rulename, sourcebatch in pairs(target:sourcebatches()) do @@ -116,13 +115,17 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, end -- get batch jobs -function _get_batchjobs(targetname, group_pattern, filepatterns) +function _get_batchjobs(targetname, opt) + + -- convert all sourcefiles to lua pattern + local filepatterns = _get_file_patterns(opt.sourcefiles) -- get root targets local targets_root = {} if targetname then table.insert(targets_root, project.target(targetname)) else + local group_pattern = opt.group_pattern local depset = hashset.new() local targets = {} for _, target in pairs(project.targets()) do @@ -195,22 +198,19 @@ function _get_file_patterns(sourcefiles) end -- the main entry -function main(targetname, group_pattern, sourcefiles) +function main(targetname, opt) - -- prepare to build - prepare_build(targetnames, {group_pattern = group_pattern, sourcefiles = sourcefiles}) - - -- convert all sourcefiles to lua pattern - local filepatterns = _get_file_patterns(sourcefiles) + -- prepare to build files + prepare_build_files(targetnames, opt) -- build all jobs - local batchjobs = _get_batchjobs(targetname, group_pattern, filepatterns) + local batchjobs = _get_batchjobs(targetname, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) os.cd(curdir) else - wprint("%s not found!", sourcefiles) + wprint("%s not found!", opt.sourcefiles) end end diff --git a/xmake/actions/build/main.lua b/xmake/actions/build/main.lua index fe7fcc771..5a6aa1535 100644 --- a/xmake/actions/build/main.lua +++ b/xmake/actions/build/main.lua @@ -101,20 +101,18 @@ function _do_project_rules(scriptname, opt) end -- do build -function _do_build(targetname, group_pattern) +function _do_build(targetname, opt) local sourcefiles = option.get("files") if sourcefiles then - build_files(targetname, group_pattern, sourcefiles) + build_files(targetname, {group_pattern = opt.group_pattern, sourcefiles = sourcefiles}) else - build(targetname, group_pattern) + build(targetname, {group_pattern = opt.group_pattern}) end end -- build targets function build_targets(targetnames, opt) opt = opt or {} - - local group_pattern = opt.group_pattern try { function () @@ -123,7 +121,7 @@ function build_targets(targetnames, opt) _do_project_rules("build_before") -- do build - _do_build(targetnames, group_pattern) + _do_build(targetnames, opt) -- do check check_targets(targetnames, {build = true}) @@ -146,8 +144,8 @@ function build_targets(targetnames, opt) -- raise if errors then raise(errors) - elseif group_pattern then - raise("build targets with group(%s) failed!", group_pattern) + elseif opt.group_pattern then + raise("build targets with group(%s) failed!", opt.group_pattern) elseif targetnames then targetnames = table.wrap(targetnames) raise("build target: %s failed!", table.concat(targetnames, ", ")) diff --git a/xmake/actions/build/prepare_files.lua b/xmake/actions/build/prepare_files.lua new file mode 100644 index 000000000..acdc41b0e --- /dev/null +++ b/xmake/actions/build/prepare_files.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_files.lua +-- + +-- imports +import("core.base.option") +import("core.project.config") + +function main(targetnames, opt) +end -- cgit v1.3.1 From 812d8ec053376a541193dc943362797fd2c4541e Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 23 Mar 2025 23:35:51 +0800 Subject: fix build files --- xmake/actions/build/build_files.lua | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 8848962db..1b29cd6c0 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -115,15 +115,28 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, end -- get batch jobs -function _get_batchjobs(targetname, opt) +function _get_batchjobs(targetnames, opt) -- convert all sourcefiles to lua pattern local filepatterns = _get_file_patterns(opt.sourcefiles) -- get root targets local targets_root = {} - if targetname then - table.insert(targets_root, project.target(targetname)) + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end else local group_pattern = opt.group_pattern local depset = hashset.new() @@ -140,6 +153,14 @@ function _get_batchjobs(targetname, opt) for _, target in pairs(targets) do if not depset:has(target:name()) then table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end end end end @@ -198,13 +219,13 @@ function _get_file_patterns(sourcefiles) end -- the main entry -function main(targetname, opt) +function main(targetnames, opt) -- prepare to build files prepare_build_files(targetnames, opt) -- build all jobs - local batchjobs = _get_batchjobs(targetname, opt) + local batchjobs = _get_batchjobs(targetnames, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) -- cgit v1.3.1 From 502f7b6ddf39b8053c523d7a58532e0a5998c414 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 24 Mar 2025 22:49:41 +0800 Subject: add target_utils --- xmake/actions/build/build.lua | 42 +---------------- xmake/actions/build/build_files.lua | 45 +------------------ xmake/actions/build/prepare.lua | 87 ++++++++++++++++++++++++++++++++++++ xmake/actions/build/target_utils.lua | 73 ++++++++++++++++++++++++++++++ xmake/modules/async/jobgraph.lua | 5 +++ 5 files changed, 169 insertions(+), 83 deletions(-) create mode 100644 xmake/actions/build/target_utils.lua diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index d11b1841b..e890faa0a 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -29,6 +29,7 @@ import("core.base.hashset") import("private.service.remote_cache.client", {alias = "remote_cache_client"}) import("private.service.distcc_build.client", {alias = "distcc_build_client"}) import("prepare", {alias = "prepare_build"}) +import("target_utils") -- clean target for rebuilding function _clean_target(target) @@ -234,46 +235,7 @@ end function get_batchjobs(targetnames, opt) -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end + local targets_root = target_utils.get_root_targets(targetnames, opt) -- generate batch jobs for default or all targets local jobrefs = {} diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 1b29cd6c0..ae09fc838 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -26,6 +26,7 @@ import("core.project.project") import("private.async.jobpool") import("async.runjobs") import("kinds.object") +import("target_utils") import("prepare_files", {alias = "prepare_build_files"}) -- match source files @@ -121,49 +122,7 @@ function _get_batchjobs(targetnames, opt) local filepatterns = _get_file_patterns(opt.sourcefiles) -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in pairs(project.targets()) do - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - for _, target in pairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - end + local targets_root = target_utils.get_root_targets(targetnames, opt) -- generate batch jobs for default or all targets local jobrefs = {} diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 52a0152af..45f0d75b9 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -21,6 +21,93 @@ -- imports import("core.base.option") import("core.project.config") +import("async.runjobs") +import("async.jobgraph", {alias = "async_jobgraph"}) + +-- get prepare jobs +function _get_prepare_jobs(targetnames, opt) + local jobgraph = async_jobgraph.new() + return jobgraph + + -- get root targets + --[[ + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + + -- generate batch jobs for default or all targets + local jobrefs = {} + local jobrefs_before = {} + local jobgraph = jobpool.new() + for _, target in ipairs(targets_root) do + _add_jobgraph_for_target_and_deps(jobgraph, jobgraph:rootjob(), target, jobrefs, jobrefs_before) + end + + -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 + for _, target in ipairs(project.ordertargets()) do + local target_job_before = jobrefs_before[target:name()] + if target_job_before then + for _, dep in ipairs(target:orderdeps()) do + if dep:policy("build.fence") then + local fence_job = jobrefs[dep:name()] + if fence_job then + jobgraph:add(fence_job, target_job_before) + end + end + end + end + end + + return jobgraph]] +end function main(targetnames, opt) + local jobgraph = _get_prepare_jobs(targetnames, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + runjobs("prepare", jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir}) + os.cd(curdir) + end end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua new file mode 100644 index 000000000..2c098400d --- /dev/null +++ b/xmake/actions/build/target_utils.lua @@ -0,0 +1,73 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file target_utils.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.config") +import("core.project.project") + +-- get all root targets +function get_root_targets(targetnames, opt) + opt = opt or {} + + -- get root targets + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + return targets_root +end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index cc4d0c743..3767cfe73 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -176,6 +176,11 @@ function jobgraph:size() return self._size end +-- is empty? +function jobgraph:empty() + return self:size() == 0 +end + -- tostring function jobgraph:__tostring() return string.format("", self:name() or "anonymous", self:size()) -- cgit v1.3.1 From 0a01105ef8e83039523a4aa97cb3ca0c8726df04 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 24 Mar 2025 22:50:28 +0800 Subject: pass target roots as args --- xmake/actions/build/build.lua | 14 ++-- xmake/actions/build/build_files.lua | 12 ++-- xmake/actions/build/prepare.lua | 47 +------------- xmake/actions/build/prepare_files.lua | 2 +- xmake/modules/private/diagnosis/dump_buildjobs.lua | 30 --------- xmake/modules/private/diagnosis/dump_targets.lua | 74 ---------------------- 6 files changed, 17 insertions(+), 162 deletions(-) delete mode 100644 xmake/modules/private/diagnosis/dump_buildjobs.lua delete mode 100644 xmake/modules/private/diagnosis/dump_targets.lua diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index e890faa0a..05284490d 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -231,11 +231,8 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, end end --- get batch jobs, @note we need to export it for private.diagnosis.dump_buildjobs -function get_batchjobs(targetnames, opt) - - -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) +-- get batch jobs +function _get_batchjobs(targets_root, opt) -- generate batch jobs for default or all targets local jobrefs = {} @@ -265,8 +262,11 @@ end function main(targetnames, opt) + -- get root targets + local targets_root = target_utils.get_root_targets(targetnames, opt) + -- prepare to build - prepare_build(targetnames, opt) + prepare_build(targets_root, opt) -- enable distcc? local distcc @@ -275,7 +275,7 @@ function main(targetnames, opt) end -- build all jobs - local batchjobs = get_batchjobs(targetnames, opt) + local batchjobs = _get_batchjobs(targets_root, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build", batchjobs, {on_exit = function (errors) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index ae09fc838..8d1383cbb 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -116,14 +116,11 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, end -- get batch jobs -function _get_batchjobs(targetnames, opt) +function _get_batchjobs(targets_root, opt) -- convert all sourcefiles to lua pattern local filepatterns = _get_file_patterns(opt.sourcefiles) - -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) - -- generate batch jobs for default or all targets local jobrefs = {} local batchjobs = jobpool.new() @@ -180,11 +177,14 @@ end -- the main entry function main(targetnames, opt) + -- get root targets + local targets_root = target_utils.get_root_targets(targetnames, opt) + -- prepare to build files - prepare_build_files(targetnames, opt) + prepare_build_files(targets_root, opt) -- build all jobs - local batchjobs = _get_batchjobs(targetnames, opt) + local batchjobs = _get_batchjobs(targets_root, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 45f0d75b9..aea951168 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -25,52 +25,11 @@ import("async.runjobs") import("async.jobgraph", {alias = "async_jobgraph"}) -- get prepare jobs -function _get_prepare_jobs(targetnames, opt) +function _get_prepare_jobs(targets_root, opt) local jobgraph = async_jobgraph.new() return jobgraph - -- get root targets --[[ - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end -- generate batch jobs for default or all targets local jobrefs = {} @@ -98,8 +57,8 @@ function _get_prepare_jobs(targetnames, opt) return jobgraph]] end -function main(targetnames, opt) - local jobgraph = _get_prepare_jobs(targetnames, opt) +function main(targets_root, opt) + local jobgraph = _get_prepare_jobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() runjobs("prepare", jobgraph, {on_exit = function (errors) diff --git a/xmake/actions/build/prepare_files.lua b/xmake/actions/build/prepare_files.lua index acdc41b0e..3abb3ef31 100644 --- a/xmake/actions/build/prepare_files.lua +++ b/xmake/actions/build/prepare_files.lua @@ -22,5 +22,5 @@ import("core.base.option") import("core.project.config") -function main(targetnames, opt) +function main(targets_root, opt) end diff --git a/xmake/modules/private/diagnosis/dump_buildjobs.lua b/xmake/modules/private/diagnosis/dump_buildjobs.lua deleted file mode 100644 index 16083a40b..000000000 --- a/xmake/modules/private/diagnosis/dump_buildjobs.lua +++ /dev/null @@ -1,30 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file dump_buildjobs.lua --- - --- imports -import("core.project.config") -import("actions.build.build", {rootdir = os.programdir()}) - --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - print(build.get_batchjobs(targetname)) -end - diff --git a/xmake/modules/private/diagnosis/dump_targets.lua b/xmake/modules/private/diagnosis/dump_targets.lua deleted file mode 100644 index c992a28f0..000000000 --- a/xmake/modules/private/diagnosis/dump_targets.lua +++ /dev/null @@ -1,74 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file dump_targets.lua --- - --- imports -import("core.base.hashset") -import("core.project.config") -import("core.project.project") - --- get targets -function _get_targets(targetname) - - -- get targets - local targets = {} - if targetname then - table.insert(targets, project.target(targetname)) - else - for _, target in pairs(project.targets()) do - table.insert(targets, target) - end - end - return targets -end - --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - for _, target in ipairs(_get_targets(targetname)) do - cprint("${bright}target(%s):${clear} %s", target:name(), target:kind()) - local deps = target:get("deps") - if deps then - cprint(" ${color.dump.string}deps:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(deps), ", ")) - end - local options = {} - for _, optname in ipairs(target:get("options")) do - if not optname:startswith("__") then - table.insert(options, optname) - end - end - if #options > 0 then - cprint(" ${color.dump.string}options:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(options), ", ")) - end - local packages = target:get("packages") - if packages then - cprint(" ${color.dump.string}packages:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(packages), ", ")) - end - local rules = target:get("rules") - if rules then - cprint(" ${color.dump.string}rules:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(rules), ", ")) - end - print("") - end -end - -- cgit v1.3.1 From 085bc689412b300623a6bddf6899213359f5f823 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 25 Mar 2025 23:54:49 +0800 Subject: walk prepare targets --- xmake/actions/build/prepare.lua | 49 ++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index aea951168..13ffa4e60 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -20,41 +20,36 @@ -- imports import("core.base.option") -import("core.project.config") +import("core.project.project") import("async.runjobs") import("async.jobgraph", {alias = "async_jobgraph"}) +-- add prepare jobs for the given target +function _add_prepare_jobs_for_target(jobgraph, target) + print("prepare", target:fullname()) +end + +-- add prepare jobs for the given target and deps +function _add_preprare_jobs_for_target_and_deps(jobgraph, target, targetrefs) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + _add_prepare_jobs_for_target(jobgraph, target) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + _add_preprare_jobs_for_target_and_deps(jobgraph, dep, targetrefs) + end + end +end + -- get prepare jobs function _get_prepare_jobs(targets_root, opt) local jobgraph = async_jobgraph.new() - return jobgraph - - --[[ - - -- generate batch jobs for default or all targets - local jobrefs = {} - local jobrefs_before = {} - local jobgraph = jobpool.new() + local targetrefs = {} for _, target in ipairs(targets_root) do - _add_jobgraph_for_target_and_deps(jobgraph, jobgraph:rootjob(), target, jobrefs, jobrefs_before) + _add_preprare_jobs_for_target_and_deps(jobgraph, target, targetrefs) end - - -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 - for _, target in ipairs(project.ordertargets()) do - local target_job_before = jobrefs_before[target:name()] - if target_job_before then - for _, dep in ipairs(target:orderdeps()) do - if dep:policy("build.fence") then - local fence_job = jobrefs[dep:name()] - if fence_job then - jobgraph:add(fence_job, target_job_before) - end - end - end - end - end - - return jobgraph]] + return jobgraph end function main(targets_root, opt) -- cgit v1.3.1 From 05605de8581d8f4f343b8e297b4e828b0b03c445 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 26 Mar 2025 23:36:37 +0800 Subject: add preparecmd apis --- xmake/actions/build/prepare.lua | 5 ++++- xmake/core/project/rule.lua | 21 +++++++++++++++------ xmake/core/project/target.lua | 27 ++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 13ffa4e60..2b2f605b2 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -26,7 +26,10 @@ import("async.jobgraph", {alias = "async_jobgraph"}) -- add prepare jobs for the given target function _add_prepare_jobs_for_target(jobgraph, target) - print("prepare", target:fullname()) + if not target:is_enabled() then + return + end + end -- add prepare jobs for the given target and deps diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index a483943d8..63f954580 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -297,13 +297,16 @@ function rule.apis() , "rule.on_clean" , "rule.on_package" , "rule.on_install" - , "rule.on_installcmd" , "rule.on_uninstall" - , "rule.on_uninstallcmd" + , "rule.on_preparecmd" + , "rule.on_preparecmd_file" + , "rule.on_preparecmd_files" , "rule.on_linkcmd" , "rule.on_buildcmd" , "rule.on_buildcmd_file" , "rule.on_buildcmd_files" + , "rule.on_installcmd" + , "rule.on_uninstallcmd" -- rule.before_xxx , "rule.before_run" , "rule.before_test" @@ -319,13 +322,16 @@ function rule.apis() , "rule.before_clean" , "rule.before_package" , "rule.before_install" - , "rule.before_installcmd" , "rule.before_uninstall" - , "rule.before_uninstallcmd" + , "rule.before_preparecmd" + , "rule.before_preparecmd_file" + , "rule.before_preparecmd_files" , "rule.before_linkcmd" , "rule.before_buildcmd" , "rule.before_buildcmd_file" , "rule.before_buildcmd_files" + , "rule.before_installcmd" + , "rule.before_uninstallcmd" -- rule.after_xxx , "rule.after_run" , "rule.after_test" @@ -341,13 +347,16 @@ function rule.apis() , "rule.after_clean" , "rule.after_package" , "rule.after_install" - , "rule.after_installcmd" , "rule.after_uninstall" - , "rule.after_uninstallcmd" + , "rule.after_preparecmd" + , "rule.after_preparecmd_file" + , "rule.after_preparecmd_files" , "rule.after_linkcmd" , "rule.after_buildcmd" , "rule.after_buildcmd_file" , "rule.after_buildcmd_files" + , "rule.after_installcmd" + , "rule.after_uninstallcmd" } } end diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 2dd779f15..089fd1e3c 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -2933,8 +2933,15 @@ function target.apis() , "target.on_clean" , "target.on_package" , "target.on_install" - , "target.on_installcmd" , "target.on_uninstall" + , "target.on_preparecmd" + , "target.on_preparecmd_file" + , "target.on_preparecmd_files" + , "target.on_linkcmd" + , "target.on_buildcmd" + , "target.on_buildcmd_file" + , "target.on_buildcmd_files" + , "target.on_installcmd" , "target.on_uninstallcmd" -- target.before_xxx , "target.before_run" @@ -2950,8 +2957,15 @@ function target.apis() , "target.before_clean" , "target.before_package" , "target.before_install" - , "target.before_installcmd" , "target.before_uninstall" + , "target.before_preparecmd" + , "target.before_preparecmd_file" + , "target.before_preparecmd_files" + , "target.before_linkcmd" + , "target.before_buildcmd" + , "target.before_buildcmd_file" + , "target.before_buildcmd_files" + , "target.before_installcmd" , "target.before_uninstallcmd" -- target.after_xxx , "target.after_run" @@ -2968,8 +2982,15 @@ function target.apis() , "target.after_clean" , "target.after_package" , "target.after_install" - , "target.after_installcmd" , "target.after_uninstall" + , "target.after_preparecmd" + , "target.after_preparecmd_file" + , "target.after_preparecmd_files" + , "target.after_linkcmd" + , "target.after_buildcmd" + , "target.after_buildcmd_file" + , "target.after_buildcmd_files" + , "target.after_installcmd" , "target.after_uninstallcmd" } } -- cgit v1.3.1 From ac5adead4c78cd74f9e3127e423fefaad700a1d8 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 26 Mar 2025 23:40:41 +0800 Subject: add runjobs for target utils --- xmake/actions/build/prepare.lua | 50 +++--------------------------------- xmake/actions/build/target_utils.lua | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 2b2f605b2..55e97afe1 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -19,52 +19,10 @@ -- -- imports -import("core.base.option") -import("core.project.project") -import("async.runjobs") -import("async.jobgraph", {alias = "async_jobgraph"}) - --- add prepare jobs for the given target -function _add_prepare_jobs_for_target(jobgraph, target) - if not target:is_enabled() then - return - end - -end - --- add prepare jobs for the given target and deps -function _add_preprare_jobs_for_target_and_deps(jobgraph, target, targetrefs) - local targetname = target:fullname() - if not targetrefs[targetname] then - targetrefs[targetname] = target - _add_prepare_jobs_for_target(jobgraph, target) - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - _add_preprare_jobs_for_target_and_deps(jobgraph, dep, targetrefs) - end - end -end - --- get prepare jobs -function _get_prepare_jobs(targets_root, opt) - local jobgraph = async_jobgraph.new() - local targetrefs = {} - for _, target in ipairs(targets_root) do - _add_preprare_jobs_for_target_and_deps(jobgraph, target, targetrefs) - end - return jobgraph -end +import("target_utils") function main(targets_root, opt) - local jobgraph = _get_prepare_jobs(targets_root, opt) - if jobgraph and not jobgraph:empty() then - local curdir = os.curdir() - runjobs("prepare", jobgraph, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir}) - os.cd(curdir) - end + opt = opt or {} + opt.jobkind = "prepare" + target_utils.runjobs(targets_root, opt) end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 2c098400d..baa63d5b2 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -23,6 +23,38 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") +import("async.runjobs") +import("async.jobgraph", {alias = "async_jobgraph"}) + +-- add jobs for the given target +function _add_jobs_for_target(jobgraph, target) + if not target:is_enabled() then + return + end +end + +-- add jobs for the given target and deps +function _add_jobs_for_target_and_deps(jobgraph, target, targetrefs) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + _add_jobs_for_target(jobgraph, target) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + _add_jobs_for_target_and_deps(jobgraph, dep, targetrefs) + end + end +end + +-- get jobs +function _get_jobs(targets_root, opt) + local jobgraph = async_jobgraph.new() + local targetrefs = {} + for _, target in ipairs(targets_root) do + _add_jobs_for_target_and_deps(jobgraph, target, targetrefs) + end + return jobgraph +end -- get all root targets function get_root_targets(targetnames, opt) @@ -71,3 +103,19 @@ function get_root_targets(targetnames, opt) end return targets_root end + +function runjobs(targets_root, opt) + opt = opt or {} + local jobkind = opt.jobkind + local jobgraph = _get_jobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + runjobs(jobkind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir}) + os.cd(curdir) + end +end -- cgit v1.3.1 From 01733eaae58b4aec4374868bcab18ee8c95c56ae Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:02:40 +0800 Subject: use dynamic script --- xmake/actions/build/prepare.lua | 2 +- xmake/actions/build/target_utils.lua | 45 +++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 55e97afe1..30207d614 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -23,6 +23,6 @@ import("target_utils") function main(targets_root, opt) opt = opt or {} - opt.jobkind = "prepare" + opt.job_kind = "prepare" target_utils.runjobs(targets_root, opt) end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index baa63d5b2..6a29f2e78 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -23,25 +23,54 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") -import("async.runjobs") +import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) +import("private.utils.batchcmds") -- add jobs for the given target -function _add_jobs_for_target(jobgraph, target) +function _add_jobs_for_target(jobgraph, target, opt) + opt = opt or {} if not target:is_enabled() then return end + + -- add after_xxx jobs for target + local job_kind = opt.job_kind + local job_after = target:fullname() .. "/after_" .. job_kind + --[[ + jobgraph:add(job_after, function (index, total, opt) + local progress = opt.progress + local script_aftername = job_kind .. "_after" + local script_after = target:script(script_aftername) + if script_after then + script_after(target, {progress = progress}) + end + for _, r in ipairs(target:orderules()) do + local script_after = r:script(script_aftername) + if script_after then + script_after(target, {progress = progress}) + else + local scriptcmd_aftername = job_kind .. "cmd_after" + local scriptcmd_after = r:script(scriptcmd_aftername) + if scriptcmd_after then + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_after(target, batchcmds_, {progress = progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + end + end)]] end -- add jobs for the given target and deps -function _add_jobs_for_target_and_deps(jobgraph, target, targetrefs) +function _add_jobs_for_target_and_deps(jobgraph, target, targetrefs, opt) local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target - _add_jobs_for_target(jobgraph, target) + _add_jobs_for_target(jobgraph, target, opt) for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) - _add_jobs_for_target_and_deps(jobgraph, dep, targetrefs) + _add_jobs_for_target_and_deps(jobgraph, dep, targetrefs, opt) end end end @@ -51,7 +80,7 @@ function _get_jobs(targets_root, opt) local jobgraph = async_jobgraph.new() local targetrefs = {} for _, target in ipairs(targets_root) do - _add_jobs_for_target_and_deps(jobgraph, target, targetrefs) + _add_jobs_for_target_and_deps(jobgraph, target, targetrefs, opt) end return jobgraph end @@ -106,11 +135,11 @@ end function runjobs(targets_root, opt) opt = opt or {} - local jobkind = opt.jobkind + local job_kind = opt.job_kind local jobgraph = _get_jobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() - runjobs(jobkind, jobgraph, {on_exit = function (errors) + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) import("utils.progress") if errors and progress.showing_without_scroll() then print("") -- cgit v1.3.1 From 4fee5fe56d3eedb3871ee237b083bb3bcbf6ec5c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:41:57 +0800 Subject: fix jobgraph for isolated vertex --- xmake/actions/build/target_utils.lua | 4 ++-- xmake/core/base/graph.lua | 11 +++++++++++ xmake/modules/async/jobgraph.lua | 4 +++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 6a29f2e78..74c57a2c9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -37,7 +37,7 @@ function _add_jobs_for_target(jobgraph, target, opt) -- add after_xxx jobs for target local job_kind = opt.job_kind local job_after = target:fullname() .. "/after_" .. job_kind - --[[ + -- TODO we should remove root job, use group instead of it jobgraph:add(job_after, function (index, total, opt) local progress = opt.progress local script_aftername = job_kind .. "_after" @@ -59,7 +59,7 @@ function _add_jobs_for_target(jobgraph, target, opt) end end end - end)]] + end) end -- add jobs for the given target and deps diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 658b71e73..43b43ce3c 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -94,6 +94,17 @@ function graph:has_vertex(v) return table.contains(self:vertices(), v) end +-- add an isolated without edges +function graph:add_vertex(v) + if not self:has_vertex(v) then + table.insert(self._vertices, v) + self._adjacent_edges[v] = {} + end + + -- reset partial topological sort state since graph structure changed + self._partial_topo_dirty = true +end + -- remove the given vertex? function graph:remove_vertex(v) local contains = false diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3767cfe73..d737c7348 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -70,10 +70,12 @@ end -- function jobgraph:add(name, run, opt) opt = opt or {} + local dag = self._dag local jobs = self._jobs if not jobs[name] then local job = {name = name, run = run, opt = opt} jobs[name] = job + dag:add_vertex(job) self._size = self._size + 1 if opt.groups then @@ -91,9 +93,9 @@ end -- remove a given job function jobgraph:remove(name) + local dag = self._dag local jobs = self._jobs local job = jobs[name] - local dag = self._dag if job then assert(self._size > 0) jobs[name] = nil -- cgit v1.3.1 From 78db72c72e2f2d1c699c7c7010e6bd84cdbf9644 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:48:17 +0800 Subject: add stage jobs for target --- xmake/actions/build/target_utils.lua | 43 ++++++++++++++++---- xmake/modules/async/jobgraph.lua | 77 ++++++++++++++++++++---------------- 2 files changed, 77 insertions(+), 43 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 74c57a2c9..aa6f06aa9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,17 +27,30 @@ import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") --- add jobs for the given target -function _add_jobs_for_target(jobgraph, target, opt) +-- add stage jobs for the given target +-- stage: before, after or "" +function _add_stage_jobs_for_target(jobgraph, target, stage, opt) opt = opt or {} - if not target:is_enabled() then - return + local job_kind = opt.job_kind + + -- the stage group, e.g. foo/after_prepare, bar/before_build + local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) + + -- call target script first, e.g. before/after_prepare, before/after_build + local progress = opt.progress + local script_name = job_kind .. "_" .. stage + local script = target:script(script_name) + if script then + local jobname = target:fullname() .. "/" .. script_name + jobgraph:add(jobname, function (index, total, opt) + -- TODO bind target envs + script(target, {progress = progress}) + end, {groups = group_name}) end - -- add after_xxx jobs for target - local job_kind = opt.job_kind - local job_after = target:fullname() .. "/after_" .. job_kind + -- TODO we should remove root job, use group instead of it + --[[ jobgraph:add(job_after, function (index, total, opt) local progress = opt.progress local script_aftername = job_kind .. "_after" @@ -59,7 +72,21 @@ function _add_jobs_for_target(jobgraph, target, opt) end end end - end) + end)]] +end + +-- add jobs for the given target +function _add_jobs_for_target(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + -- add group jobs for target, e.g. after_xxx -> (depend on) on_xxx -> before_xxx + local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) + local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) + local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) + jobgraph:add_deps(group_after, group, group_before) end -- add jobs for the given target and deps diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index d737c7348..3440f606b 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -105,54 +105,61 @@ function jobgraph:remove(name) end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +-- +-- and it supports nil, e.g add_deps("foo", nil, "bar", ...) +-- function jobgraph:add_deps(...) local prev local prev_is_group local dag = self._dag local jobs = self._jobs local groups = self._groups - for _, name in ipairs(table.pack(...)) do - local curr_is_group = false - local curr = jobs[name] - if not curr then - curr = groups[name] - curr_is_group = true - end - assert(curr, "job(%s) not found in jobgraph(%s)", name, self) - if prev then - if prev_is_group and curr_is_group then - -- we use a fake job as a node to bridge the two groups. - local fakejob = {} - for _, job in ipairs(prev) do - if not dag:has_edge(job, fakejob) then - dag:add_edge(job, fakejob) + local deps = table.pack(...) + for i = 1, deps.n do + local name = deps[i] + if name then + local curr_is_group = false + local curr = jobs[name] + if not curr then + curr = groups[name] + curr_is_group = true + end + assert(curr, "job(%s) not found in jobgraph(%s)", name, self) + if prev then + if prev_is_group and curr_is_group then + -- we use a fake job as a node to bridge the two groups. + local fakejob = {} + for _, job in ipairs(prev) do + if not dag:has_edge(job, fakejob) then + dag:add_edge(job, fakejob) + end end - end - for _, job in ipairs(curr) do - if not dag:has_edge(fakejob, job) then - dag:add_edge(fakejob, job) + for _, job in ipairs(curr) do + if not dag:has_edge(fakejob, job) then + dag:add_edge(fakejob, job) + end end - end - elseif curr_is_group then - for _, job in ipairs(curr) do - if not dag:has_edge(prev, job) then - dag:add_edge(prev, job) + elseif curr_is_group then + for _, job in ipairs(curr) do + if not dag:has_edge(prev, job) then + dag:add_edge(prev, job) + end end - end - elseif prev_is_group then - for _, job in ipairs(prev) do - if not dag:has_edge(job, curr) then - dag:add_edge(job, curr) + elseif prev_is_group then + for _, job in ipairs(prev) do + if not dag:has_edge(job, curr) then + dag:add_edge(job, curr) + end + end + else + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) end - end - else - if not dag:has_edge(prev, curr) then - dag:add_edge(prev, curr) end end + prev = curr + prev_is_group = curr_is_group end - prev = curr - prev_is_group = curr_is_group end end -- cgit v1.3.1 From 37c593a71339240ea880ab5478f2f5120af992cc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:54:11 +0800 Subject: improve jobs --- xmake/actions/build/target_utils.lua | 62 +++++++++++++++++------------------- xmake/modules/async/jobgraph.lua | 8 ++++- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index aa6f06aa9..aa5413995 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -32,47 +32,45 @@ import("private.utils.batchcmds") function _add_stage_jobs_for_target(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind + local progress = opt.progress - -- the stage group, e.g. foo/after_prepare, bar/before_build + -- the group name, e.g. foo/after_prepare, bar/before_build local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) - -- call target script first, e.g. before/after_prepare, before/after_build - local progress = opt.progress - local script_name = job_kind .. "_" .. stage - local script = target:script(script_name) - if script then - local jobname = target:fullname() .. "/" .. script_name - jobgraph:add(jobname, function (index, total, opt) - -- TODO bind target envs - script(target, {progress = progress}) - end, {groups = group_name}) - end + -- the script name, e.g. before/after_prepare, before/after_build + local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind + -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd + local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - -- TODO we should remove root job, use group instead of it - --[[ - jobgraph:add(job_after, function (index, total, opt) - local progress = opt.progress - local script_aftername = job_kind .. "_after" - local script_after = target:script(script_aftername) - if script_after then - script_after(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local script_after = r:script(script_aftername) - if script_after then - script_after(target, {progress = progress}) - else - local scriptcmd_aftername = job_kind .. "cmd_after" - local scriptcmd_after = r:script(scriptcmd_aftername) - if scriptcmd_after then + -- call target and rules script + local jobdeps = {} + local instances = table.join(target, target:orderules()) -- TODO sort them + for _, instance in ipairs(instances) do + local script = instance:script(script_name) + if script then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + -- TODO bind target envs + script(target, {progress = progress}) + end, {groups = group_name}) + table.insert(jobdeps, jobname) + else + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_after(target, batchcmds_, {progress = progress}) + scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end + end, {groups = group_name}) + table.insert(jobdeps, jobname) end end - end)]] + end + + -- add job deps + jobgraph:add_deps(jobdeps) end -- add jobs for the given target diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3440f606b..b72fb5b16 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -107,6 +107,7 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... -- -- and it supports nil, e.g add_deps("foo", nil, "bar", ...) +-- and it also supports to add deps list, e.g. add_deps(deps) -- function jobgraph:add_deps(...) local prev @@ -115,7 +116,12 @@ function jobgraph:add_deps(...) local jobs = self._jobs local groups = self._groups local deps = table.pack(...) - for i = 1, deps.n do + local count = deps.n + if count == 1 and type(deps[1]) == "table" then + deps = deps[1] + count = #deps + end + for i = 1, count do local name = deps[i] if name then local curr_is_group = false -- cgit v1.3.1 From 881671b9516d2e5b76436db1c9456aa02625ec52 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:57:36 +0800 Subject: fix instances list --- xmake/actions/build/target_utils.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index aa5413995..1fef599ac 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -43,9 +43,14 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") + -- TODO sort them + local instances = {target} + for _, r in ipairs(target:orderules()) do + table.insert(instances, r) + end + -- call target and rules script local jobdeps = {} - local instances = table.join(target, target:orderules()) -- TODO sort them for _, instance in ipairs(instances) do local script = instance:script(script_name) if script then @@ -60,6 +65,7 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) if scriptcmd then local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) + -- TODO bind target envs local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) -- cgit v1.3.1 From 8beb0732ff716cd5b3d7a40744b1d239302fdb4b Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 22:49:41 +0800 Subject: rename add_deps to add_orders --- tests/modules/async/run_jobgraph.lua | 4 ++-- xmake/actions/build/target_utils.lua | 15 +++++++++------ xmake/core/base/graph.lua | 18 +++++++++++------- xmake/modules/async/jobgraph.lua | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index bc1b5500f..9b8f529ed 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -18,7 +18,7 @@ function _test_basic() jobs:add("job/" .. i, _jobfunc) for j = 1, 50 do jobs:add("job/" .. i .. "/" .. j, _jobfunc) - jobs:add_deps("job/" .. i .. "/" .. j, "job/" .. i, "job/root") + jobs:add_orders("job/" .. i .. "/" .. j, "job/" .. i, "job/root") end end t = os.mclock() @@ -37,7 +37,7 @@ function _test_group() jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) end end - jobs:add_deps("foo", "bar", "job/root") + jobs:add_orders("foo", "bar", "job/root") t = os.mclock() runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 1fef599ac..b91b436f0 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -50,7 +50,7 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) end -- call target and rules script - local jobdeps = {} + local joborders = {} for _, instance in ipairs(instances) do local script = instance:script(script_name) if script then @@ -59,7 +59,7 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- TODO bind target envs script(target, {progress = progress}) end, {groups = group_name}) - table.insert(jobdeps, jobname) + table.insert(joborders, jobname) else local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then @@ -70,13 +70,16 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end, {groups = group_name}) - table.insert(jobdeps, jobname) + table.insert(joborders, jobname) end end end - -- add job deps - jobgraph:add_deps(jobdeps) + -- add job orders + if #joborders > 0 then + jobgraph:add_orders(joborders) + return group_name + end end -- add jobs for the given target @@ -90,7 +93,7 @@ function _add_jobs_for_target(jobgraph, target, opt) local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) - jobgraph:add_deps(group_after, group, group_before) + jobgraph:add_orders(group_before, group, group_after) end -- add jobs for the given target and deps diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 43b43ce3c..db8682b21 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -173,12 +173,14 @@ end -- -- e.g. -- --- add_edge(a, b) -- a depend on b --- add_edge(b, c) -- b depend on c +-- edges: a (indegree: 0) -> b -> c -- --- local node1, has_cycle = g:partial_topo_sort_next() -- return c +-- add_edge(a, b) +-- add_edge(b, c) +-- +-- local node1, has_cycle = g:partial_topo_sort_next() -- return a -- local node2, has_cycle = g:partial_topo_sort_next() -- return b --- local node3, has_cycle = g:partial_topo_sort_next() -- return a +-- local node3, has_cycle = g:partial_topo_sort_next() -- return c -- local node4, has_cycle = g:partial_topo_sort_next() -- return nil (empty, all done) -- function graph:partial_topo_sort_next() @@ -250,10 +252,12 @@ end -- -- e.g. -- --- add_edge(a, b) -- a depend on b --- add_edge(b, c) -- b depend on c +-- edges: a (indegree: 0) -> b -> c +-- +-- add_edge(a, b) +-- add_edge(b, c) -- --- it will return {c, b, a} +-- it will return {a, b, c} function graph:topo_sort() if not self:is_directed() then return diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index b72fb5b16..fe42f6647 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -104,25 +104,25 @@ function jobgraph:remove(name) end end --- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +-- add job orders, e.g. add_orders(a, b, c, ...): a -> b -> c, ... -- --- and it supports nil, e.g add_deps("foo", nil, "bar", ...) --- and it also supports to add deps list, e.g. add_deps(deps) +-- and it supports nil, e.g add_orders("foo", nil, "bar", ...) +-- and it also supports to add orders list, e.g. add_orders(orders) -- -function jobgraph:add_deps(...) +function jobgraph:add_orders(...) local prev local prev_is_group local dag = self._dag local jobs = self._jobs local groups = self._groups - local deps = table.pack(...) - local count = deps.n - if count == 1 and type(deps[1]) == "table" then - deps = deps[1] - count = #deps + local orders = table.pack(...) + local count = orders.n + if count == 1 and type(orders[1]) == "table" then + orders = orders[1] + count = #orders end for i = 1, count do - local name = deps[i] + local name = orders[i] if name then local curr_is_group = false local curr = jobs[name] -- cgit v1.3.1 From e10063b886379b0bb2d28666c23e0eefd9cda9d7 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 22:54:56 +0800 Subject: bind target envs --- xmake/actions/build/target_utils.lua | 59 ++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index b91b436f0..d92d548c7 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,6 +27,14 @@ import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + -- add stage jobs for the given target -- stage: before, after or "" function _add_stage_jobs_for_target(jobgraph, target, stage, opt) @@ -56,7 +64,6 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) if script then local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) jobgraph:add(jobname, function (index, total, opt) - -- TODO bind target envs script(target, {progress = progress}) end, {groups = group_name}) table.insert(joborders, jobname) @@ -65,7 +72,6 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) if scriptcmd then local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) - -- TODO bind target envs local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) @@ -89,11 +95,58 @@ function _add_jobs_for_target(jobgraph, target, opt) return end + local pkgenvs = _g.pkgenvs + if pkgenvs == nil then + pkgenvs = {} + _g.pkgenvs = pkgenvs + end + + local job_kind = opt.job_kind + local job_begin = string.format("target/%s/begin_%s", target:fullname(), job_kind) + local job_end = string.format("target/%s/end_%s", target:fullname(), job_kind) + jobgraph:add(job_begin, function (index, total, opt) + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target first if rebuild + if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + end) + + jobgraph:add(job_end, function (index, total, opt) + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + end) + -- add group jobs for target, e.g. after_xxx -> (depend on) on_xxx -> before_xxx local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) - jobgraph:add_orders(group_before, group, group_after) + jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end -- add jobs for the given target and deps -- cgit v1.3.1 From e12ddee3cc3d237b6f01f03c71f18763781eb1eb Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 23:02:06 +0800 Subject: add script job --- xmake/actions/build/target_utils.lua | 74 ++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index d92d548c7..41c979b85 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -35,12 +35,61 @@ function _clean_target(target) end end +-- add script job +function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) + opt = opt or {} + local joborders = opt.joborders + local group_name = opt.group_name + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end, {groups = group_name}) + table.insert(joborders, jobname) + end + else + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {groups = group_name}) + table.insert(joborders, jobname) + end + end +end + -- add stage jobs for the given target -- stage: before, after or "" function _add_stage_jobs_for_target(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind - local progress = opt.progress -- the group name, e.g. foo/after_prepare, bar/before_build local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) @@ -60,25 +109,10 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- call target and rules script local joborders = {} for _, instance in ipairs(instances) do - local script = instance:script(script_name) - if script then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = progress}) - end, {groups = group_name}) - table.insert(joborders, jobname) - else - local scriptcmd = instance:script(scriptcmd_name) - if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {groups = group_name}) - table.insert(joborders, jobname) - end - end + _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { + group_name = group_name, + joborders = joborders + }) end -- add job orders -- cgit v1.3.1 From f55ca39e18390317210ac7d660cbfaa517c1e691 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 23:05:03 +0800 Subject: add jobgraph:group --- tests/modules/async/run_jobgraph.lua | 8 +++++--- xmake/actions/build/target_utils.lua | 21 ++++++++++++--------- xmake/modules/async/jobgraph.lua | 27 +++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index 9b8f529ed..4463038b7 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -33,9 +33,11 @@ function _test_group() jobs:add("job/root", _jobfunc) for i = 1, 3 do jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) - for j = 1, 50 do - jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) - end + jobgraph:group("foo", function () + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + end + end) end jobs:add_orders("foo", "bar", "job/root") t = os.mclock() diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 41c979b85..13cf54ad9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -39,7 +39,6 @@ end function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) opt = opt or {} local joborders = opt.joborders - local group_name = opt.group_name local script = instance:script(script_name) if script then -- call custom script with jobgraph @@ -49,6 +48,7 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) -- on_build(function (target, jobgraph, opt) -- end, {jobgraph = true}) if instance:extraconf(script_name, "jobgraph") then + -- TODO group and joborders script(target, jobgraph) elseif instance:extraconf(script_name, "batch") then wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it.", instance:fullname(), script_name) @@ -62,9 +62,11 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) jobgraph:add(jobname, function (index, total, opt) script(target, {progress = opt.progress}) - end, {groups = group_name}) + end) table.insert(joborders, jobname) end + elseif false then + -- TODO call builtin script else -- call command script -- e.g. @@ -79,7 +81,7 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = opt.progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {groups = group_name}) + end) table.insert(joborders, jobname) end end @@ -108,12 +110,13 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- call target and rules script local joborders = {} - for _, instance in ipairs(instances) do - _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { - group_name = group_name, - joborders = joborders - }) - end + jobgraph:group(group_name, function () + for _, instance in ipairs(instances) do + _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { + joborders = joborders + }) + end + end) -- add job orders if #joborders > 0 then diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index fe42f6647..b9e9a46da 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -78,8 +78,9 @@ function jobgraph:add(name, run, opt) dag:add_vertex(job) self._size = self._size + 1 - if opt.groups then - for _, group_name in ipairs(opt.groups) do + if self._current_groups or opt.groups then + local job_groups = table.join(self._current_groups or {}, opt.groups) + for _, group_name in ipairs(job_groups) do local groups = self._groups[group_name] if not groups then groups = {} @@ -104,6 +105,28 @@ function jobgraph:remove(name) end end +-- enter group to add jobs +-- +-- e.g. +-- jobgraph:group("foo", function () +-- jobgraph:add("job1", function (index, total, opt) +-- TODO +-- end) +-- jobgraph:add("job2", function (index, total, opt) +-- TODO +-- end) +-- end) +function jobgraph:group(name, callback) + local current_groups = self._current_groups + if current_groups == nil then + current_groups = {} + self._current_groups = current_groups + end + table.insert(current_groups, name) + callback() + table.remove(current_groups) +end + -- add job orders, e.g. add_orders(a, b, c, ...): a -> b -> c, ... -- -- and it supports nil, e.g add_orders("foo", nil, "bar", ...) -- cgit v1.3.1 From 987ab86a6270073e6d1d08d4e8e70d6487e0d65c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 23:05:19 +0800 Subject: fix test --- tests/modules/async/run_jobgraph.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index 4463038b7..fe53a54c6 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -33,7 +33,7 @@ function _test_group() jobs:add("job/root", _jobfunc) for i = 1, 3 do jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) - jobgraph:group("foo", function () + jobs:group("foo", function () for j = 1, 50 do jobs:add("job/" .. i .. "/" .. j, _jobfunc) end -- cgit v1.3.1 From 6582a3e7eb62f1c6ad4a8263bd14579e7a25c092 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 23:19:54 +0800 Subject: add builtin jobs --- xmake/actions/build/target_utils.lua | 57 +++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 13cf54ad9..dd6328b23 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -35,10 +35,19 @@ function _clean_target(target) end end --- add script job -function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) - opt = opt or {} - local joborders = opt.joborders +-- add jobs for the builtin script +function _add_jobs_for_builtin_script(jobgraph, target, job_kind) + if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then + local script = import("kinds." .. target:kind(), {anonymous = true})[job_kind] + if script then + script(jobgraph, target) + end + end +end + +-- add jobs for the given script +function _add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) + local has_script = false local script = instance:script(script_name) if script then -- call custom script with jobgraph @@ -48,7 +57,6 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) -- on_build(function (target, jobgraph, opt) -- end, {jobgraph = true}) if instance:extraconf(script_name, "jobgraph") then - -- TODO group and joborders script(target, jobgraph) elseif instance:extraconf(script_name, "batch") then wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it.", instance:fullname(), script_name) @@ -63,10 +71,8 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) jobgraph:add(jobname, function (index, total, opt) script(target, {progress = opt.progress}) end) - table.insert(joborders, jobname) end - elseif false then - -- TODO call builtin script + has_script = true else -- call command script -- e.g. @@ -82,14 +88,15 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) scriptcmd(target, batchcmds_, {progress = opt.progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end) - table.insert(joborders, jobname) + has_script = true end end + return has_script end --- add stage jobs for the given target +-- add jobs for the given target stage -- stage: before, after or "" -function _add_stage_jobs_for_target(jobgraph, target, stage, opt) +function _add_jobs_for_target_stage(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind @@ -102,25 +109,29 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - -- TODO sort them + -- TODO sort rules and jobs local instances = {target} for _, r in ipairs(target:orderules()) do table.insert(instances, r) end -- call target and rules script - local joborders = {} + local jobsize = jobgraph:size() jobgraph:group(group_name, function () + local has_script = false for _, instance in ipairs(instances) do - _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { - joborders = joborders - }) + if _add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then + has_script = true + end + end + + -- call builtin script, e.g. on_prepare, on_build, ... + if not has_script and stage == "" then + _add_jobs_for_builtin_script(jobgraph, target, job_kind) end end) - -- add job orders - if #joborders > 0 then - jobgraph:add_orders(joborders) + if jobgraph:size() > jobsize then return group_name end end @@ -179,10 +190,10 @@ function _add_jobs_for_target(jobgraph, target, opt) end end) - -- add group jobs for target, e.g. after_xxx -> (depend on) on_xxx -> before_xxx - local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) - local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) - local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) + -- add jobs for target stage, e.g. after_xxx -> (depend on) on_xxx -> before_xxx + local group = _add_jobs_for_target_stage(jobgraph, target, "", opt) + local group_before = _add_jobs_for_target_stage(jobgraph, target, "before", opt) + local group_after = _add_jobs_for_target_stage(jobgraph, target, "after", opt) jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end -- cgit v1.3.1 From 7e60e473a3d92d0706a4856c12f161c4d2df32a9 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 28 Mar 2025 00:37:04 +0800 Subject: mark as jobbatch as deprecated --- xmake/actions/build/.DS_Store | Bin 0 -> 6148 bytes xmake/actions/build/build.lua | 266 ++----------------- xmake/actions/build/build_files.lua | 162 +----------- xmake/actions/build/deprecated/.DS_Store | Bin 0 -> 6148 bytes xmake/actions/build/deprecated/build.lua | 283 +++++++++++++++++++++ xmake/actions/build/deprecated/build_files.lua | 188 ++++++++++++++ xmake/actions/build/deprecated/kinds/binary.lua | 152 +++++++++++ .../build/deprecated/kinds/linkdepfiles.lua | 39 +++ .../actions/build/deprecated/kinds/moduleonly.lua | 22 ++ xmake/actions/build/deprecated/kinds/object.lua | 273 ++++++++++++++++++++ xmake/actions/build/deprecated/kinds/shared.lua | 153 +++++++++++ xmake/actions/build/deprecated/kinds/static.lua | 153 +++++++++++ xmake/actions/build/kinds/binary.lua | 152 ----------- xmake/actions/build/kinds/linkdepfiles.lua | 39 --- xmake/actions/build/kinds/moduleonly.lua | 22 -- xmake/actions/build/kinds/object.lua | 273 -------------------- xmake/actions/build/kinds/shared.lua | 153 ----------- xmake/actions/build/kinds/static.lua | 153 ----------- xmake/actions/build/prepare.lua | 28 -- xmake/actions/build/prepare_files.lua | 26 -- xmake/actions/build/target_utils.lua | 5 +- xmake/core/project/policy.lua | 2 + 22 files changed, 1292 insertions(+), 1252 deletions(-) create mode 100644 xmake/actions/build/.DS_Store create mode 100644 xmake/actions/build/deprecated/.DS_Store create mode 100644 xmake/actions/build/deprecated/build.lua create mode 100644 xmake/actions/build/deprecated/build_files.lua create mode 100644 xmake/actions/build/deprecated/kinds/binary.lua create mode 100644 xmake/actions/build/deprecated/kinds/linkdepfiles.lua create mode 100644 xmake/actions/build/deprecated/kinds/moduleonly.lua create mode 100644 xmake/actions/build/deprecated/kinds/object.lua create mode 100644 xmake/actions/build/deprecated/kinds/shared.lua create mode 100644 xmake/actions/build/deprecated/kinds/static.lua delete mode 100644 xmake/actions/build/kinds/binary.lua delete mode 100644 xmake/actions/build/kinds/linkdepfiles.lua delete mode 100644 xmake/actions/build/kinds/moduleonly.lua delete mode 100644 xmake/actions/build/kinds/object.lua delete mode 100644 xmake/actions/build/kinds/shared.lua delete mode 100644 xmake/actions/build/kinds/static.lua delete mode 100644 xmake/actions/build/prepare.lua delete mode 100644 xmake/actions/build/prepare_files.lua diff --git a/xmake/actions/build/.DS_Store b/xmake/actions/build/.DS_Store new file mode 100644 index 000000000..b82482eba Binary files /dev/null and b/xmake/actions/build/.DS_Store differ diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 05284490d..54827d354 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -22,242 +22,20 @@ import("core.base.option") import("core.project.config") import("core.project.project") -import("private.async.jobpool") -import("async.runjobs") -import("private.utils.batchcmds") -import("core.base.hashset") -import("private.service.remote_cache.client", {alias = "remote_cache_client"}) -import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -import("prepare", {alias = "prepare_build"}) import("target_utils") +import("deprecated.build", {alias = "deprecated_build"}) --- clean target for rebuilding -function _clean_target(target) - if target:targetfile() then - os.tryrm(target:symbolfile()) - os.tryrm(target:targetfile()) - end -end - --- add builtin batch jobs -function _add_batchjobs_builtin(batchjobs, rootjob, target) - - -- uses the rules script? - local job, job_leaf - for _, r in irpairs(target:orderules()) do -- reverse rules order for batchjobs:addjob() - local script = r:script("build") - if script then - if r:extraconf("build", "batch") then - job, job_leaf = assert(script(target, batchjobs, {rootjob = job or rootjob}), "rule(%s):on_build(): no returned job!", r:name()) - else - job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) - script(target, {progress = opt.progress}) - end, {rootjob = job or rootjob}) - end - else - local buildcmd = r:script("buildcmd") - if buildcmd then - job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - buildcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = job or rootjob}) - end - end - end - - -- uses the builtin target script - if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly()) then - job, job_leaf = import("kinds." .. target:kind(), {anonymous = true})(batchjobs, rootjob, target) - end - job = job or rootjob - return job, job_leaf or job +function _prepare(targets_root, opt) + opt = opt or {} + opt.job_kind = "prepare" + target_utils.runjobs(targets_root, opt) end --- add batch jobs -function _add_batchjobs(batchjobs, rootjob, target) - - local job, job_leaf - local script = target:script("build") - if not script then - -- do builtin batch jobs - job, job_leaf = _add_batchjobs_builtin(batchjobs, rootjob, target) - elseif target:extraconf("build", "batch") then - -- do custom batch script - -- e.g. - -- target("test") - -- on_build(function (target, batchjobs, opt) - -- return batchjobs:addjob("test", function (idx, total) - -- print("build it") - -- end, {rootjob = opt.rootjob}) - -- end, {batch = true}) - -- - job, job_leaf = assert(script(target, batchjobs, {rootjob = rootjob}), "target(%s):on_build(): no returned job!", target:name()) - else - -- do custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- print("build it") - -- end) - -- - job = batchjobs:addjob(target:name() .. "/build", function (index, total, opt) - script(target, {progress = opt.progress}) - end, {rootjob = rootjob}) - end - return job, job_leaf or job -end - --- add batch jobs for the given target -function _add_batchjobs_for_target(batchjobs, rootjob, target) - - -- has been disabled? - if not target:is_enabled() then - return - end - - -- add after_build job for target - local pkgenvs = _g.pkgenvs or {} - _g.pkgenvs = pkgenvs - local job_build_after = batchjobs:addjob(target:name() .. "/after_build", function (index, total, opt) - - -- do after_build - local progress = opt.progress - local after_build = target:script("build_after") - if after_build then - after_build(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local after_build = r:script("build_after") - if after_build then - after_build(target, {progress = progress}) - else - local after_buildcmd = r:script("buildcmd_after") - if after_buildcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_buildcmd(target, batchcmds_, {progress = progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - end - - -- restore environments - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = nil - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - end, {rootjob = rootjob}) - - -- add batch jobs for target, @note only on_build script support batch jobs - local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_build_after, target) - - -- add before_build job for target - local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total, opt) - - -- enter package environments - -- https://github.com/xmake-io/xmake/issues/4033 - -- - -- maybe mixing envs isn't a great solution, - -- but it's the most efficient compromise compared to setting envs in every on_build_file. - -- - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = target:pkgenvs() - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - -- clean target if rebuild - if target:is_rebuilt() and not option.get("dry-run") then - _clean_target(target) - end - - -- do before_build - -- we cannot add batchjobs for this rule scripts, @see https://github.com/xmake-io/xmake/issues/2684 - local progress = opt.progress - local before_build = target:script("build_before") - if before_build then - before_build(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local before_build = r:script("build_before") - if before_build then - before_build(target, {progress = progress}) - else - local before_buildcmd = r:script("buildcmd_before") - if before_buildcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_buildcmd(target, batchcmds_, {progress = progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - end - end, {rootjob = job_build_leaf}) - return job_build_before, job_build, job_build_after -end - --- add batch jobs for the given target and deps -function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, jobrefs_before) - local targetjob_ref = jobrefs[target:name()] - if targetjob_ref then - batchjobs:add(targetjob_ref, rootjob) - else - local job_build_before, job_build, job_build_after = _add_batchjobs_for_target(batchjobs, rootjob, target) - if job_build_before and job_build and job_build_after then - jobrefs[target:name()] = job_build_after - jobrefs_before[target:name()] = job_build_before - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - local targetjob = job_build - -- @see https://github.com/xmake-io/xmake/discussions/2500 - if dep:policy("build.across_targets_in_parallel") == false then - targetjob = job_build_before - end - _add_batchjobs_for_target_and_deps(batchjobs, targetjob, dep, jobrefs, jobrefs_before) - end - end - end -end - --- get batch jobs -function _get_batchjobs(targets_root, opt) - - -- generate batch jobs for default or all targets - local jobrefs = {} - local jobrefs_before = {} - local batchjobs = jobpool.new() - for _, target in ipairs(targets_root) do - _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), target, jobrefs, jobrefs_before) - end - - -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 - for _, target in ipairs(project.ordertargets()) do - local target_job_before = jobrefs_before[target:name()] - if target_job_before then - for _, dep in ipairs(target:orderdeps()) do - if dep:policy("build.fence") then - local fence_job = jobrefs[dep:name()] - if fence_job then - batchjobs:add(fence_job, target_job_before) - end - end - end - end - end - - return batchjobs +function _build(targets_root, opt) + --[[ + opt = opt or {} + opt.job_kind = "build" + target_utils.runjobs(targets_root, opt)]] end function main(targetnames, opt) @@ -266,24 +44,12 @@ function main(targetnames, opt) local targets_root = target_utils.get_root_targets(targetnames, opt) -- prepare to build - prepare_build(targets_root, opt) - - -- enable distcc? - local distcc - if distcc_build_client.is_connected() then - distcc = distcc_build_client.singleton() - end + _prepare(targets_root, opt) - -- build all jobs - local batchjobs = _get_batchjobs(targets_root, opt) - if batchjobs and batchjobs:size() > 0 then - local curdir = os.curdir() - runjobs("build", batchjobs, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = distcc}) - os.cd(curdir) + -- do build + if project.policy("build.jobgraph") then + _build(targets_root, opt) + else + deprecated_build(targets_root, opt) end end diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 8d1383cbb..718390e0b 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -23,174 +23,28 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") -import("private.async.jobpool") -import("async.runjobs") -import("kinds.object") import("target_utils") -import("prepare_files", {alias = "prepare_build_files"}) +import("deprecated.build_files", {alias = "deprecated_build_files"}) --- match source files -function _match_sourcefiles(sourcefile, filepatterns) - for _, filepattern in ipairs(filepatterns) do - if sourcefile:match(filepattern.pattern) == sourcefile then - if filepattern.excludes then - if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then - sourcefile = sourcefile:sub(#filepattern.rootdir + 2) - end - for _, exclude in ipairs(filepattern.excludes) do - if sourcefile:match(exclude) == sourcefile then - return false - end - end - end - return true - end - end -end - --- add batch jobs -function _add_batchjobs(batchjobs, rootjob, target, filepatterns) - local newbatches = {} - local sourcecount = 0 - for rulename, sourcebatch in pairs(target:sourcebatches()) do - local objectfiles = sourcebatch.objectfiles - local dependfiles = sourcebatch.dependfiles - local sourcekind = sourcebatch.sourcekind - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do - if _match_sourcefiles(sourcefile, filepatterns) then - local newbatch = newbatches[rulename] - if not newbatch then - newbatch = {} - newbatch.sourcekind = sourcekind - newbatch.rulename = rulename - newbatch.sourcefiles = {} - end - table.insert(newbatch.sourcefiles, sourcefile) - if objectfiles then - newbatch.objectfiles = newbatch.objectfiles or {} - table.insert(newbatch.objectfiles, objectfiles[idx]) - end - if dependfiles then - newbatch.dependfiles = newbatch.dependfiles or {} - table.insert(newbatch.dependfiles, dependfiles[idx]) - end - newbatches[rulename] = newbatch - sourcecount = sourcecount + 1 - end - end - end - if sourcecount > 0 then - return object.add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, newbatches) - end -end - --- add batch jobs for the given target -function _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) - - -- has been disabled? - if not target:is_enabled() then - return - end - - -- add batch jobs for target - return _add_batchjobs(batchjobs, rootjob, target, filepatterns) -end - --- add batch jobs for the given target and deps -function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, filepatterns) - local targetjob_ref = jobrefs[target:name()] - if targetjob_ref then - batchjobs:add(targetjob_ref, rootjob) - else - local targetjob, targetjob_root = _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) - if targetjob and targetjob_root then - jobrefs[target:name()] = targetjob_root - if not option.get("shallow") then - for _, depname in ipairs(target:get("deps")) do - _add_batchjobs_for_target_and_deps(batchjobs, targetjob, jobrefs, - project.target(depname, {namespace = target:namespace()}), filepatterns) - end - end - end - end +function _prepare_files(targets_root, opt) end --- get batch jobs -function _get_batchjobs(targets_root, opt) - - -- convert all sourcefiles to lua pattern - local filepatterns = _get_file_patterns(opt.sourcefiles) - - -- generate batch jobs for default or all targets - local jobrefs = {} - local batchjobs = jobpool.new() - for _, target in pairs(targets_root) do - _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), jobrefs, target, filepatterns) - end - return batchjobs -end - --- convert all sourcefiles to lua pattern -function _get_file_patterns(sourcefiles) - local patterns = {} - for _, sourcefile in ipairs(path.splitenv(sourcefiles)) do - - -- get the excludes - local pattern = sourcefile:trim() - local excludes = pattern:match("|.*$") - if excludes then excludes = excludes:split("|", {plain = true}) end - - -- translate excludes - if excludes then - local _excludes = {} - for _, exclude in ipairs(excludes) do - exclude = path.translate(exclude) - exclude = path.pattern(exclude) - table.insert(_excludes, exclude) - end - excludes = _excludes - end - - -- translate path and remove some repeat separators - pattern = path.translate(pattern:gsub("|.*$", "")) - - -- remove "./" or '.\\' prefix - if pattern:sub(1, 2):find('%.[/\\]') then - pattern = pattern:sub(3) - end - - -- get the root directory - local rootdir = pattern - local startpos = pattern:find("*", 1, true) - if startpos then - rootdir = rootdir:sub(1, startpos - 1) - end - rootdir = path.directory(rootdir) - - -- convert to lua path pattern - pattern = path.pattern(pattern) - table.insert(patterns, {pattern = pattern, excludes = excludes, rootdir = rootdir}) - end - return patterns +function _build_files(targets_root, opt) end --- the main entry function main(targetnames, opt) -- get root targets local targets_root = target_utils.get_root_targets(targetnames, opt) -- prepare to build files - prepare_build_files(targets_root, opt) + _prepare_files(targets_root, opt) - -- build all jobs - local batchjobs = _get_batchjobs(targets_root, opt) - if batchjobs and batchjobs:size() > 0 then - local curdir = os.curdir() - runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) - os.cd(curdir) + -- do build files + if project.policy("build.jobgraph") then + _build_files(targets_root, opt) else - wprint("%s not found!", opt.sourcefiles) + deprecated_build_files(targets_root, opt) end end diff --git a/xmake/actions/build/deprecated/.DS_Store b/xmake/actions/build/deprecated/.DS_Store new file mode 100644 index 000000000..a8ae5f9d6 Binary files /dev/null and b/xmake/actions/build/deprecated/.DS_Store differ diff --git a/xmake/actions/build/deprecated/build.lua b/xmake/actions/build/deprecated/build.lua new file mode 100644 index 000000000..1c8d47a18 --- /dev/null +++ b/xmake/actions/build/deprecated/build.lua @@ -0,0 +1,283 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build.lua +-- + +-- imports +import("core.base.option") +import("core.project.config") +import("core.project.project") +import("private.async.jobpool") +import("async.runjobs") +import("private.utils.batchcmds") +import("core.base.hashset") +import("private.service.remote_cache.client", {alias = "remote_cache_client"}) +import("private.service.distcc_build.client", {alias = "distcc_build_client"}) + +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + +-- add builtin batch jobs +function _add_batchjobs_builtin(batchjobs, rootjob, target) + + -- uses the rules script? + local job, job_leaf + for _, r in irpairs(target:orderules()) do -- reverse rules order for batchjobs:addjob() + local script = r:script("build") + if script then + if r:extraconf("build", "batch") then + job, job_leaf = assert(script(target, batchjobs, {rootjob = job or rootjob}), "rule(%s):on_build(): no returned job!", r:name()) + elseif r:extraconf("build", "jobgraph") then + wprint("rule(%s) with jobgraph found, please enable `build.jobgraph` policy first!", r:name()) + else + job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) + script(target, {progress = opt.progress}) + end, {rootjob = job or rootjob}) + end + else + local buildcmd = r:script("buildcmd") + if buildcmd then + job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + buildcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = job or rootjob}) + end + end + end + + -- uses the builtin target script + if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly()) then + job, job_leaf = import("kinds." .. target:kind(), {anonymous = true})(batchjobs, rootjob, target) + end + job = job or rootjob + return job, job_leaf or job +end + +-- add batch jobs +function _add_batchjobs(batchjobs, rootjob, target) + + local job, job_leaf + local script = target:script("build") + if not script then + -- do builtin batch jobs + job, job_leaf = _add_batchjobs_builtin(batchjobs, rootjob, target) + elseif target:extraconf("build", "batch") then + -- do custom batch script + -- e.g. + -- target("test") + -- on_build(function (target, batchjobs, opt) + -- return batchjobs:addjob("test", function (idx, total) + -- print("build it") + -- end, {rootjob = opt.rootjob}) + -- end, {batch = true}) + -- + job, job_leaf = assert(script(target, batchjobs, {rootjob = rootjob}), "target(%s):on_build(): no returned job!", target:name()) + else + -- do custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- print("build it") + -- end) + -- + job = batchjobs:addjob(target:name() .. "/build", function (index, total, opt) + script(target, {progress = opt.progress}) + end, {rootjob = rootjob}) + end + return job, job_leaf or job +end + +-- add batch jobs for the given target +function _add_batchjobs_for_target(batchjobs, rootjob, target) + + -- has been disabled? + if not target:is_enabled() then + return + end + + -- add after_build job for target + local pkgenvs = _g.pkgenvs or {} + _g.pkgenvs = pkgenvs + local job_build_after = batchjobs:addjob(target:name() .. "/after_build", function (index, total, opt) + + -- do after_build + local progress = opt.progress + local after_build = target:script("build_after") + if after_build then + after_build(target, {progress = progress}) + end + for _, r in ipairs(target:orderules()) do + local after_build = r:script("build_after") + if after_build then + after_build(target, {progress = progress}) + else + local after_buildcmd = r:script("buildcmd_after") + if after_buildcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_buildcmd(target, batchcmds_, {progress = progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + end + + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + end, {rootjob = rootjob}) + + -- add batch jobs for target, @note only on_build script support batch jobs + local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_build_after, target) + + -- add before_build job for target + local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total, opt) + + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target if rebuild + if target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + + -- do before_build + -- we cannot add batchjobs for this rule scripts, @see https://github.com/xmake-io/xmake/issues/2684 + local progress = opt.progress + local before_build = target:script("build_before") + if before_build then + before_build(target, {progress = progress}) + end + for _, r in ipairs(target:orderules()) do + local before_build = r:script("build_before") + if before_build then + before_build(target, {progress = progress}) + else + local before_buildcmd = r:script("buildcmd_before") + if before_buildcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_buildcmd(target, batchcmds_, {progress = progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + end + end, {rootjob = job_build_leaf}) + return job_build_before, job_build, job_build_after +end + +-- add batch jobs for the given target and deps +function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, jobrefs_before) + local targetjob_ref = jobrefs[target:name()] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local job_build_before, job_build, job_build_after = _add_batchjobs_for_target(batchjobs, rootjob, target) + if job_build_before and job_build and job_build_after then + jobrefs[target:name()] = job_build_after + jobrefs_before[target:name()] = job_build_before + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + local targetjob = job_build + -- @see https://github.com/xmake-io/xmake/discussions/2500 + if dep:policy("build.across_targets_in_parallel") == false then + targetjob = job_build_before + end + _add_batchjobs_for_target_and_deps(batchjobs, targetjob, dep, jobrefs, jobrefs_before) + end + end + end +end + +-- get batch jobs +function _get_batchjobs(targets_root, opt) + + -- generate batch jobs for default or all targets + local jobrefs = {} + local jobrefs_before = {} + local batchjobs = jobpool.new() + for _, target in ipairs(targets_root) do + _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), target, jobrefs, jobrefs_before) + end + + -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 + for _, target in ipairs(project.ordertargets()) do + local target_job_before = jobrefs_before[target:name()] + if target_job_before then + for _, dep in ipairs(target:orderdeps()) do + if dep:policy("build.fence") then + local fence_job = jobrefs[dep:name()] + if fence_job then + batchjobs:add(fence_job, target_job_before) + end + end + end + end + end + + return batchjobs +end + +function main(targets_root, opt) + + -- enable distcc? + local distcc + if distcc_build_client.is_connected() then + distcc = distcc_build_client.singleton() + end + + -- build all jobs + local batchjobs = _get_batchjobs(targets_root, opt) + if batchjobs and batchjobs:size() > 0 then + local curdir = os.curdir() + runjobs("build", batchjobs, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = distcc}) + os.cd(curdir) + end +end diff --git a/xmake/actions/build/deprecated/build_files.lua b/xmake/actions/build/deprecated/build_files.lua new file mode 100644 index 000000000..db58385cb --- /dev/null +++ b/xmake/actions/build/deprecated/build_files.lua @@ -0,0 +1,188 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_files.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.config") +import("core.project.project") +import("private.async.jobpool") +import("async.runjobs") +import("kinds.object") + +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- add batch jobs +function _add_batchjobs(batchjobs, rootjob, target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return object.add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, newbatches) + end +end + +-- add batch jobs for the given target +function _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) + + -- has been disabled? + if not target:is_enabled() then + return + end + + -- add batch jobs for target + return _add_batchjobs(batchjobs, rootjob, target, filepatterns) +end + +-- add batch jobs for the given target and deps +function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, filepatterns) + local targetjob_ref = jobrefs[target:name()] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local targetjob, targetjob_root = _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) + if targetjob and targetjob_root then + jobrefs[target:name()] = targetjob_root + if not option.get("shallow") then + for _, depname in ipairs(target:get("deps")) do + _add_batchjobs_for_target_and_deps(batchjobs, targetjob, jobrefs, + project.target(depname, {namespace = target:namespace()}), filepatterns) + end + end + end + end +end + +-- get batch jobs +function _get_batchjobs(targets_root, opt) + + -- convert all sourcefiles to lua pattern + local filepatterns = _get_file_patterns(opt.sourcefiles) + + -- generate batch jobs for default or all targets + local jobrefs = {} + local batchjobs = jobpool.new() + for _, target in pairs(targets_root) do + _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), jobrefs, target, filepatterns) + end + return batchjobs +end + +-- convert all sourcefiles to lua pattern +function _get_file_patterns(sourcefiles) + local patterns = {} + for _, sourcefile in ipairs(path.splitenv(sourcefiles)) do + + -- get the excludes + local pattern = sourcefile:trim() + local excludes = pattern:match("|.*$") + if excludes then excludes = excludes:split("|", {plain = true}) end + + -- translate excludes + if excludes then + local _excludes = {} + for _, exclude in ipairs(excludes) do + exclude = path.translate(exclude) + exclude = path.pattern(exclude) + table.insert(_excludes, exclude) + end + excludes = _excludes + end + + -- translate path and remove some repeat separators + pattern = path.translate(pattern:gsub("|.*$", "")) + + -- remove "./" or '.\\' prefix + if pattern:sub(1, 2):find('%.[/\\]') then + pattern = pattern:sub(3) + end + + -- get the root directory + local rootdir = pattern + local startpos = pattern:find("*", 1, true) + if startpos then + rootdir = rootdir:sub(1, startpos - 1) + end + rootdir = path.directory(rootdir) + + -- convert to lua path pattern + pattern = path.pattern(pattern) + table.insert(patterns, {pattern = pattern, excludes = excludes, rootdir = rootdir}) + end + return patterns +end + +function main(targets_root, opt) + + -- build all jobs + local batchjobs = _get_batchjobs(targets_root, opt) + if batchjobs and batchjobs:size() > 0 then + local curdir = os.curdir() + runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) + os.cd(curdir) + else + wprint("%s not found!", opt.sourcefiles) + end +end + + diff --git a/xmake/actions/build/deprecated/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua new file mode 100644 index 000000000..3f67232f7 --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -0,0 +1,152 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file binary.lua +-- + +-- imports +import("core.base.option") +import("core.theme.theme") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("private.utils.batchcmds") +import("object", {alias = "object_target"}) +import("linkdepfiles", {alias = "get_linkdepfiles"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +-- on link the given target +function _on_link_target(target, opt) + + -- link target with rules + local done = false + for _, r in ipairs(target:orderules()) do + local on_link = r:script("link") + if on_link then + on_link(target, opt) + done = true + end + local on_linkcmd = r:script("linkcmd") + if on_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + on_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + done = true + end + end + if done then return end + + -- do link + _do_link_target(target, opt) +end + +-- link target +function _link_target(target, opt) + + -- do before link for target + local before_link = target:script("link_before") + if before_link then + before_link(target, opt) + end + + -- do before link for rules + for _, r in ipairs(target:orderules()) do + local before_link = r:script("link_before") + if before_link then + before_link(target, opt) + end + local before_linkcmd = r:script("linkcmd_before") + if before_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + + -- on link + target:script("link", _on_link_target)(target, opt) + + -- do after link for target + local after_link = target:script("link_after") + if after_link then + after_link(target, opt) + end + + -- do after link for rules + for _, r in ipairs(target:orderules()) do + local after_link = r:script("link_after") + if after_link then + after_link(target, opt) + end + local after_linkcmd = r:script("linkcmd_after") + if after_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end +end + +-- add batch jobs for building binary target +function main(batchjobs, rootjob, target) + + -- add link job + local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) + _link_target(target, {progress = opt.progress}) + end, {rootjob = rootjob}) + + -- we only need to return and depend the link job for each target, + -- so we can compile the source files for each target in parallel + -- + -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. + -- + local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) + return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects +end diff --git a/xmake/actions/build/deprecated/kinds/linkdepfiles.lua b/xmake/actions/build/deprecated/kinds/linkdepfiles.lua new file mode 100644 index 000000000..f96e532fc --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/linkdepfiles.lua @@ -0,0 +1,39 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file linkdepfiles.lua +-- + +-- get link depfiles +function main(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end diff --git a/xmake/actions/build/deprecated/kinds/moduleonly.lua b/xmake/actions/build/deprecated/kinds/moduleonly.lua new file mode 100644 index 000000000..d177b8b97 --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/moduleonly.lua @@ -0,0 +1,22 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file moduleonly.lua +-- + +-- imports +inherit("object") diff --git a/xmake/actions/build/deprecated/kinds/object.lua b/xmake/actions/build/deprecated/kinds/object.lua new file mode 100644 index 000000000..b5359f79e --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/object.lua @@ -0,0 +1,273 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file object.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") +import("async.runjobs") +import("private.utils.batchcmds") +import("private.utils.rule_groups") + +-- has scripts for the custom rule +function _has_scripts_for_rule(ruleinst, suffix) + + -- add batch jobs for xx_build_files + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_build_file + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_buildcmd_files + scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_buildcmd_file + scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end +end + +-- has scripts for target +function _has_scripts_for_target(target, suffix) + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = target:script(scriptname) + if script then + return true + else + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = target:script(scriptname) + if script then + return true + end + end +end + +-- has scripts for group +function _has_scripts_for_group(group, suffix) + for _, item in pairs(group) do + if item.target and _has_scripts_for_target(item.target, suffix) then + return true + end + if item.rule and _has_scripts_for_rule(item.rule, suffix) then + return true + end + end +end + +-- add batch jobs for the custom rule +function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) + + -- get rule + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local ruleinst = rule_groups.get_rule(target, rulename) + + -- add batch jobs for xx_build_files + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = ruleinst:script(scriptname) + if script then + if ruleinst:extraconf(scriptname, "batch") then + script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + else + batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total, opt) + script(target, sourcebatch, {progress = opt.progress}) + end, {rootjob = rootjob}) + end + end + + -- add batch jobs for xx_build_file + if not script then + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total, opt) + script(target, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) + end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + end + end + end + + -- add batch jobs for xx_buildcmd_files + if not script then + scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + local distcc = ruleinst:extraconf(scriptname, "distcc") + script(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = rootjob}) + end + end + + -- add batch jobs for xx_buildcmd_file + if not script then + scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + end + end + end +end + +-- add batch jobs for target +function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) + + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = rule_groups.get_rule(target, rulename) + if not ruleinst:script("build_file") and + not ruleinst:script("build_files") then + return + end + end + + -- add batch jobs + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = target:script(scriptname) + if script then + if target:extraconf(scriptname, "batch") then + script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) + else + batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total, opt) + script(target, sourcebatch, {progress = opt.progress}) + end, {rootjob = rootjob}) + end + return true + else + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = target:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total, opt) + script(target, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) + end, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) + end + return true + end + end +end + +-- add batch jobs for group +function _add_batchjobs_for_group(batchjobs, rootjob, target, group, suffix) + for _, item in pairs(group) do + local sourcebatch = item.sourcebatch + if item.target then + _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) + end + -- override on_xxx script in target? we need to ignore rule scripts + if item.rule and (suffix or not _has_scripts_for_target(target, suffix)) then + _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) + end + end +end + +-- add batch jobs for building source files +function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches) + + -- build sourcebatch groups first + local groups = rule_groups.build_sourcebatch_groups(target, sourcebatches) + + -- add batch jobs for build_after + local groups_root + local groups_leaf = rootjob + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group, "after") then + batchjobs:group_enter(target:name() .. "/after_build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "after") + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + + -- add batch jobs for build + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group) then + batchjobs:group_enter(target:name() .. "/build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group) + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + + -- add batch jobs for build_before + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group, "before") then + batchjobs:group_enter(target:name() .. "/before_build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "before") + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + return groups_leaf, groups_root or groups_leaf +end + +-- add batch jobs for building object files +function add_batchjobs_for_object(batchjobs, rootjob, target) + return add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, target:sourcebatches()) +end + +-- add batch jobs for building object target +function main(batchjobs, rootjob, target) + + -- add a fake link job + local job_link = batchjobs:addjob(target:name() .. "/fakelink", function (index, total, opt) + end, {rootjob = rootjob}) + + -- we only need to return and depend the link job for each target, + -- so we can compile the source files for each target in parallel + -- + -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. + -- + local job_objects = add_batchjobs_for_object(batchjobs, job_link, target) + return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects +end diff --git a/xmake/actions/build/deprecated/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua new file mode 100644 index 000000000..cc7aa010c --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -0,0 +1,153 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file shared.lua +-- + +-- imports +import("core.base.option") +import("core.theme.theme") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("private.utils.batchcmds") +import("object", {alias = "object_target"}) +import("linkdepfiles", {alias = "get_linkdepfiles"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +-- on link the given target +function _on_link_target(target, opt) + + -- link target with rules + local done = false + for _, r in ipairs(target:orderules()) do + local on_link = r:script("link") + if on_link then + on_link(target, opt) + done = true + end + local on_linkcmd = r:script("linkcmd") + if on_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + on_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + done = true + end + end + if done then return end + + -- do link + _do_link_target(target, opt) +end + +-- link target +function _link_target(target, opt) + + -- do before link for target + local before_link = target:script("link_before") + if before_link then + before_link(target, opt) + end + + -- do before link for rules + for _, r in ipairs(target:orderules()) do + local before_link = r:script("link_before") + if before_link then + before_link(target, opt) + end + local before_linkcmd = r:script("linkcmd_before") + if before_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + + -- on link + target:script("link", _on_link_target)(target, opt) + + -- do after link for target + local after_link = target:script("link_after") + if after_link then + after_link(target, opt) + end + + -- do after link for rules + for _, r in ipairs(target:orderules()) do + local after_link = r:script("link_after") + if after_link then + after_link(target, opt) + end + local after_linkcmd = r:script("linkcmd_after") + if after_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end +end + +-- add batch jobs for building shared target +function main(batchjobs, rootjob, target) + + -- add link job + local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) + _link_target(target, {progress = opt.progress}) + end, {rootjob = rootjob}) + + -- we only need to return and depend the link job for each target, + -- so we can compile the source files for each target in parallel + -- + -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. + -- + local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) + return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects +end diff --git a/xmake/actions/build/deprecated/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua new file mode 100644 index 000000000..abb6c5f72 --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -0,0 +1,153 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file static.lua +-- + +-- imports +import("core.base.option") +import("core.theme.theme") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("private.utils.batchcmds") +import("object", {alias = "object_target"}) +import("linkdepfiles", {alias = "get_linkdepfiles"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}archiving.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +-- on link the given target +function _on_link_target(target, opt) + + -- link target with rules + local done = false + for _, r in ipairs(target:orderules()) do + local on_link = r:script("link") + if on_link then + on_link(target, opt) + done = true + end + local on_linkcmd = r:script("linkcmd") + if on_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + on_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + done = true + end + end + if done then return end + + -- do link + _do_link_target(target, opt) +end + +-- link target +function _link_target(target, opt) + + -- do before link for target + local before_link = target:script("link_before") + if before_link then + before_link(target, opt) + end + + -- do before link for rules + for _, r in ipairs(target:orderules()) do + local before_link = r:script("link_before") + if before_link then + before_link(target, opt) + end + local before_linkcmd = r:script("linkcmd_before") + if before_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + + -- on link + target:script("link", _on_link_target)(target, opt) + + -- do after link for target + local after_link = target:script("link_after") + if after_link then + after_link(target, opt) + end + + -- do after link for rules + for _, r in ipairs(target:orderules()) do + local after_link = r:script("link_after") + if after_link then + after_link(target, opt) + end + local after_linkcmd = r:script("linkcmd_after") + if after_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end +end + +-- add batch jobs for building static target +function main(batchjobs, rootjob, target) + + -- add link job + local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) + _link_target(target, {progress = opt.progress}) + end, {rootjob = rootjob}) + + -- we only need to return and depend the link job for each target, + -- so we can compile the source files for each target in parallel + -- + -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. + -- + local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) + return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects +end diff --git a/xmake/actions/build/kinds/binary.lua b/xmake/actions/build/kinds/binary.lua deleted file mode 100644 index 3f67232f7..000000000 --- a/xmake/actions/build/kinds/binary.lua +++ /dev/null @@ -1,152 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file binary.lua --- - --- imports -import("core.base.option") -import("core.theme.theme") -import("core.tool.linker") -import("core.tool.compiler") -import("core.project.depend") -import("utils.progress") -import("private.utils.batchcmds") -import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) - --- do link target -function _do_link_target(target, opt) - local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) - local linkflags = linkinst:linkflags({target = target}) - - -- need build this target? - local depfiles = get_linkdepfiles(target) - local dryrun = option.get("dry-run") - local depvalues = {linkinst:program(), linkflags} - depend.on_changed(function () - local filename = target:filename() - if target:namespace() then - filename = target:namespace() .. "::" .. filename - end - progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) - - local targetfile = target:targetfile() - local objectfiles = target:objectfiles() - local verbose = option.get("verbose") - if verbose then - -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows - print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) - end - - if not dryrun then - assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) - end - end, {dependfile = target:dependfile(), - lastmtime = os.mtime(target:targetfile()), - changed = target:is_rebuilt() or option.get("linkonly"), - values = depvalues, files = depfiles, dryrun = dryrun}) -end - --- on link the given target -function _on_link_target(target, opt) - - -- link target with rules - local done = false - for _, r in ipairs(target:orderules()) do - local on_link = r:script("link") - if on_link then - on_link(target, opt) - done = true - end - local on_linkcmd = r:script("linkcmd") - if on_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - on_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - done = true - end - end - if done then return end - - -- do link - _do_link_target(target, opt) -end - --- link target -function _link_target(target, opt) - - -- do before link for target - local before_link = target:script("link_before") - if before_link then - before_link(target, opt) - end - - -- do before link for rules - for _, r in ipairs(target:orderules()) do - local before_link = r:script("link_before") - if before_link then - before_link(target, opt) - end - local before_linkcmd = r:script("linkcmd_before") - if before_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - - -- on link - target:script("link", _on_link_target)(target, opt) - - -- do after link for target - local after_link = target:script("link_after") - if after_link then - after_link(target, opt) - end - - -- do after link for rules - for _, r in ipairs(target:orderules()) do - local after_link = r:script("link_after") - if after_link then - after_link(target, opt) - end - local after_linkcmd = r:script("linkcmd_after") - if after_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end -end - --- add batch jobs for building binary target -function main(batchjobs, rootjob, target) - - -- add link job - local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) - _link_target(target, {progress = opt.progress}) - end, {rootjob = rootjob}) - - -- we only need to return and depend the link job for each target, - -- so we can compile the source files for each target in parallel - -- - -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. - -- - local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) - return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects -end diff --git a/xmake/actions/build/kinds/linkdepfiles.lua b/xmake/actions/build/kinds/linkdepfiles.lua deleted file mode 100644 index f96e532fc..000000000 --- a/xmake/actions/build/kinds/linkdepfiles.lua +++ /dev/null @@ -1,39 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file linkdepfiles.lua --- - --- get link depfiles -function main(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles -end diff --git a/xmake/actions/build/kinds/moduleonly.lua b/xmake/actions/build/kinds/moduleonly.lua deleted file mode 100644 index d177b8b97..000000000 --- a/xmake/actions/build/kinds/moduleonly.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki, Arthapz --- @file moduleonly.lua --- - --- imports -inherit("object") diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua deleted file mode 100644 index b5359f79e..000000000 --- a/xmake/actions/build/kinds/object.lua +++ /dev/null @@ -1,273 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file object.lua --- - --- imports -import("core.base.option") -import("core.project.rule") -import("core.project.config") -import("core.project.project") -import("async.runjobs") -import("private.utils.batchcmds") -import("private.utils.rule_groups") - --- has scripts for the custom rule -function _has_scripts_for_rule(ruleinst, suffix) - - -- add batch jobs for xx_build_files - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_build_file - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_buildcmd_files - scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_buildcmd_file - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end -end - --- has scripts for target -function _has_scripts_for_target(target, suffix) - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = target:script(scriptname) - if script then - return true - else - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = target:script(scriptname) - if script then - return true - end - end -end - --- has scripts for group -function _has_scripts_for_group(group, suffix) - for _, item in pairs(group) do - if item.target and _has_scripts_for_target(item.target, suffix) then - return true - end - if item.rule and _has_scripts_for_rule(item.rule, suffix) then - return true - end - end -end - --- add batch jobs for the custom rule -function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) - - -- get rule - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = rule_groups.get_rule(target, rulename) - - -- add batch jobs for xx_build_files - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - if ruleinst:extraconf(scriptname, "batch") then - script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - else - batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total, opt) - script(target, sourcebatch, {progress = opt.progress}) - end, {rootjob = rootjob}) - end - end - - -- add batch jobs for xx_build_file - if not script then - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total, opt) - script(target, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) - end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - end - end - end - - -- add batch jobs for xx_buildcmd_files - if not script then - scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - local distcc = ruleinst:extraconf(scriptname, "distcc") - script(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = rootjob}) - end - end - - -- add batch jobs for xx_buildcmd_file - if not script then - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - end - end - end -end - --- add batch jobs for target -function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) - - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - local rulename = sourcebatch.rulename - if rulename then - local ruleinst = rule_groups.get_rule(target, rulename) - if not ruleinst:script("build_file") and - not ruleinst:script("build_files") then - return - end - end - - -- add batch jobs - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = target:script(scriptname) - if script then - if target:extraconf(scriptname, "batch") then - script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) - else - batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total, opt) - script(target, sourcebatch, {progress = opt.progress}) - end, {rootjob = rootjob}) - end - return true - else - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = target:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total, opt) - script(target, sourcefile, {sourcekind = sourcekind, progress = opt.progress}) - end, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) - end - return true - end - end -end - --- add batch jobs for group -function _add_batchjobs_for_group(batchjobs, rootjob, target, group, suffix) - for _, item in pairs(group) do - local sourcebatch = item.sourcebatch - if item.target then - _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) - end - -- override on_xxx script in target? we need to ignore rule scripts - if item.rule and (suffix or not _has_scripts_for_target(target, suffix)) then - _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) - end - end -end - --- add batch jobs for building source files -function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches) - - -- build sourcebatch groups first - local groups = rule_groups.build_sourcebatch_groups(target, sourcebatches) - - -- add batch jobs for build_after - local groups_root - local groups_leaf = rootjob - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group, "after") then - batchjobs:group_enter(target:name() .. "/after_build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "after") - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - - -- add batch jobs for build - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group) then - batchjobs:group_enter(target:name() .. "/build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group) - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - - -- add batch jobs for build_before - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group, "before") then - batchjobs:group_enter(target:name() .. "/before_build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "before") - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - return groups_leaf, groups_root or groups_leaf -end - --- add batch jobs for building object files -function add_batchjobs_for_object(batchjobs, rootjob, target) - return add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, target:sourcebatches()) -end - --- add batch jobs for building object target -function main(batchjobs, rootjob, target) - - -- add a fake link job - local job_link = batchjobs:addjob(target:name() .. "/fakelink", function (index, total, opt) - end, {rootjob = rootjob}) - - -- we only need to return and depend the link job for each target, - -- so we can compile the source files for each target in parallel - -- - -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. - -- - local job_objects = add_batchjobs_for_object(batchjobs, job_link, target) - return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects -end diff --git a/xmake/actions/build/kinds/shared.lua b/xmake/actions/build/kinds/shared.lua deleted file mode 100644 index cc7aa010c..000000000 --- a/xmake/actions/build/kinds/shared.lua +++ /dev/null @@ -1,153 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file shared.lua --- - --- imports -import("core.base.option") -import("core.theme.theme") -import("core.tool.linker") -import("core.tool.compiler") -import("core.project.depend") -import("utils.progress") -import("private.utils.batchcmds") -import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) - --- do link target -function _do_link_target(target, opt) - local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) - local linkflags = linkinst:linkflags({target = target}) - - -- need build this target? - local depfiles = get_linkdepfiles(target) - local dryrun = option.get("dry-run") - local depvalues = {linkinst:program(), linkflags} - depend.on_changed(function () - local filename = target:filename() - if target:namespace() then - filename = target:namespace() .. "::" .. filename - end - progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) - - local targetfile = target:targetfile() - local objectfiles = target:objectfiles() - local verbose = option.get("verbose") - if verbose then - -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows - print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) - end - - if not dryrun then - assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) - end - - end, {dependfile = target:dependfile(), - lastmtime = os.mtime(target:targetfile()), - changed = target:is_rebuilt() or option.get("linkonly"), - values = depvalues, files = depfiles, dryrun = dryrun}) -end - --- on link the given target -function _on_link_target(target, opt) - - -- link target with rules - local done = false - for _, r in ipairs(target:orderules()) do - local on_link = r:script("link") - if on_link then - on_link(target, opt) - done = true - end - local on_linkcmd = r:script("linkcmd") - if on_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - on_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - done = true - end - end - if done then return end - - -- do link - _do_link_target(target, opt) -end - --- link target -function _link_target(target, opt) - - -- do before link for target - local before_link = target:script("link_before") - if before_link then - before_link(target, opt) - end - - -- do before link for rules - for _, r in ipairs(target:orderules()) do - local before_link = r:script("link_before") - if before_link then - before_link(target, opt) - end - local before_linkcmd = r:script("linkcmd_before") - if before_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - - -- on link - target:script("link", _on_link_target)(target, opt) - - -- do after link for target - local after_link = target:script("link_after") - if after_link then - after_link(target, opt) - end - - -- do after link for rules - for _, r in ipairs(target:orderules()) do - local after_link = r:script("link_after") - if after_link then - after_link(target, opt) - end - local after_linkcmd = r:script("linkcmd_after") - if after_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end -end - --- add batch jobs for building shared target -function main(batchjobs, rootjob, target) - - -- add link job - local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) - _link_target(target, {progress = opt.progress}) - end, {rootjob = rootjob}) - - -- we only need to return and depend the link job for each target, - -- so we can compile the source files for each target in parallel - -- - -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. - -- - local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) - return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects -end diff --git a/xmake/actions/build/kinds/static.lua b/xmake/actions/build/kinds/static.lua deleted file mode 100644 index abb6c5f72..000000000 --- a/xmake/actions/build/kinds/static.lua +++ /dev/null @@ -1,153 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file static.lua --- - --- imports -import("core.base.option") -import("core.theme.theme") -import("core.tool.linker") -import("core.tool.compiler") -import("core.project.depend") -import("utils.progress") -import("private.utils.batchcmds") -import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) - --- do link target -function _do_link_target(target, opt) - local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) - local linkflags = linkinst:linkflags({target = target}) - - -- need build this target? - local depfiles = get_linkdepfiles(target) - local dryrun = option.get("dry-run") - local depvalues = {linkinst:program(), linkflags} - depend.on_changed(function () - local filename = target:filename() - if target:namespace() then - filename = target:namespace() .. "::" .. filename - end - progress.show(opt.progress, "${color.build.target}archiving.$(mode) %s", filename) - - local targetfile = target:targetfile() - local objectfiles = target:objectfiles() - local verbose = option.get("verbose") - if verbose then - -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows - print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) - end - - if not dryrun then - assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) - end - - end, {dependfile = target:dependfile(), - lastmtime = os.mtime(target:targetfile()), - changed = target:is_rebuilt() or option.get("linkonly"), - values = depvalues, files = depfiles, dryrun = dryrun}) -end - --- on link the given target -function _on_link_target(target, opt) - - -- link target with rules - local done = false - for _, r in ipairs(target:orderules()) do - local on_link = r:script("link") - if on_link then - on_link(target, opt) - done = true - end - local on_linkcmd = r:script("linkcmd") - if on_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - on_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - done = true - end - end - if done then return end - - -- do link - _do_link_target(target, opt) -end - --- link target -function _link_target(target, opt) - - -- do before link for target - local before_link = target:script("link_before") - if before_link then - before_link(target, opt) - end - - -- do before link for rules - for _, r in ipairs(target:orderules()) do - local before_link = r:script("link_before") - if before_link then - before_link(target, opt) - end - local before_linkcmd = r:script("linkcmd_before") - if before_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - - -- on link - target:script("link", _on_link_target)(target, opt) - - -- do after link for target - local after_link = target:script("link_after") - if after_link then - after_link(target, opt) - end - - -- do after link for rules - for _, r in ipairs(target:orderules()) do - local after_link = r:script("link_after") - if after_link then - after_link(target, opt) - end - local after_linkcmd = r:script("linkcmd_after") - if after_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end -end - --- add batch jobs for building static target -function main(batchjobs, rootjob, target) - - -- add link job - local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total, opt) - _link_target(target, {progress = opt.progress}) - end, {rootjob = rootjob}) - - -- we only need to return and depend the link job for each target, - -- so we can compile the source files for each target in parallel - -- - -- unless call set_policy("build.across_targets_in_parallel", false) to disable to build across targets in parallel. - -- - local job_objects = object_target.add_batchjobs_for_object(batchjobs, job_link, target) - return target:policy("build.across_targets_in_parallel") == false and job_objects or job_link, job_objects -end diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua deleted file mode 100644 index 30207d614..000000000 --- a/xmake/actions/build/prepare.lua +++ /dev/null @@ -1,28 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare.lua --- - --- imports -import("target_utils") - -function main(targets_root, opt) - opt = opt or {} - opt.job_kind = "prepare" - target_utils.runjobs(targets_root, opt) -end diff --git a/xmake/actions/build/prepare_files.lua b/xmake/actions/build/prepare_files.lua deleted file mode 100644 index 3abb3ef31..000000000 --- a/xmake/actions/build/prepare_files.lua +++ /dev/null @@ -1,26 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_files.lua --- - --- imports -import("core.base.option") -import("core.project.config") - -function main(targets_root, opt) -end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index dd6328b23..0c1c64abf 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -38,7 +38,8 @@ end -- add jobs for the builtin script function _add_jobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then - local script = import("kinds." .. target:kind(), {anonymous = true})[job_kind] + -- TODO + local script = import("deprecated.kinds." .. target:kind(), {anonymous = true})[job_kind] if script then script(jobgraph, target) end @@ -59,7 +60,7 @@ function _add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) if instance:extraconf(script_name, "jobgraph") then script(target, jobgraph) elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it.", instance:fullname(), script_name) + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) else -- call custom script directly -- e.g. diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index c3754d540..76016e164 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -94,6 +94,8 @@ function policy.policies() ["build.c++.gcc.modules.cxx11abi"] = {description = "Force to enable new cxx11 abi in C++ modules for gcc.", type = "boolean"}, -- Enable cuda device link ["build.cuda.devlink"] = {description = "Enable Cuda devlink.", type = "boolean"}, + -- Enable build jobgraph + ["build.jobgraph"] = {description = "Enable build jobgraph.", default = false, type = "boolean"}, -- Enable windows UAC and set level, e.g. invoker, admin, highest ["windows.manifest.uac"] = {description = "Enable windows manifest UAC.", type = "string"}, -- Enable ui access for windows UAC -- cgit v1.3.1 From 43c58c12e7f6a29efc19ae11d27765cfa9e301e2 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 28 Mar 2025 00:41:30 +0800 Subject: add builtin actions --- xmake/actions/build/.DS_Store | Bin 6148 -> 0 bytes xmake/actions/build/build.lua | 1 + xmake/actions/build/builtin/build_binary.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/build_moduleonly.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/build_object.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/build_shared.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/build_static.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/prepare_binary.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/prepare_moduleonly.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/prepare_object.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/prepare_shared.lua | 25 +++++++++++++++++++++ xmake/actions/build/builtin/prepare_static.lua | 25 +++++++++++++++++++++ xmake/actions/build/deprecated/.DS_Store | Bin 6148 -> 0 bytes xmake/actions/build/target_utils.lua | 3 +-- 14 files changed, 252 insertions(+), 2 deletions(-) delete mode 100644 xmake/actions/build/.DS_Store create mode 100644 xmake/actions/build/builtin/build_binary.lua create mode 100644 xmake/actions/build/builtin/build_moduleonly.lua create mode 100644 xmake/actions/build/builtin/build_object.lua create mode 100644 xmake/actions/build/builtin/build_shared.lua create mode 100644 xmake/actions/build/builtin/build_static.lua create mode 100644 xmake/actions/build/builtin/prepare_binary.lua create mode 100644 xmake/actions/build/builtin/prepare_moduleonly.lua create mode 100644 xmake/actions/build/builtin/prepare_object.lua create mode 100644 xmake/actions/build/builtin/prepare_shared.lua create mode 100644 xmake/actions/build/builtin/prepare_static.lua delete mode 100644 xmake/actions/build/deprecated/.DS_Store diff --git a/xmake/actions/build/.DS_Store b/xmake/actions/build/.DS_Store deleted file mode 100644 index b82482eba..000000000 Binary files a/xmake/actions/build/.DS_Store and /dev/null differ diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 54827d354..63121ea07 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -31,6 +31,7 @@ function _prepare(targets_root, opt) target_utils.runjobs(targets_root, opt) end +-- TODO distcc function _build(targets_root, opt) --[[ opt = opt or {} diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua new file mode 100644 index 000000000..591c679c3 --- /dev/null +++ b/xmake/actions/build/builtin/build_binary.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_binary.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/build_moduleonly.lua b/xmake/actions/build/builtin/build_moduleonly.lua new file mode 100644 index 000000000..b45766f4c --- /dev/null +++ b/xmake/actions/build/builtin/build_moduleonly.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_moduleonly.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/build_object.lua b/xmake/actions/build/builtin/build_object.lua new file mode 100644 index 000000000..0d13f392e --- /dev/null +++ b/xmake/actions/build/builtin/build_object.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_object.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/build_shared.lua b/xmake/actions/build/builtin/build_shared.lua new file mode 100644 index 000000000..a178dafd6 --- /dev/null +++ b/xmake/actions/build/builtin/build_shared.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_shared.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/build_static.lua b/xmake/actions/build/builtin/build_static.lua new file mode 100644 index 000000000..21de325da --- /dev/null +++ b/xmake/actions/build/builtin/build_static.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_static.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/prepare_binary.lua b/xmake/actions/build/builtin/prepare_binary.lua new file mode 100644 index 000000000..827f02d95 --- /dev/null +++ b/xmake/actions/build/builtin/prepare_binary.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_binary.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/prepare_moduleonly.lua b/xmake/actions/build/builtin/prepare_moduleonly.lua new file mode 100644 index 000000000..3f133c9ff --- /dev/null +++ b/xmake/actions/build/builtin/prepare_moduleonly.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_moduleonly.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/prepare_object.lua b/xmake/actions/build/builtin/prepare_object.lua new file mode 100644 index 000000000..7197d9ad5 --- /dev/null +++ b/xmake/actions/build/builtin/prepare_object.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_object.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/prepare_shared.lua b/xmake/actions/build/builtin/prepare_shared.lua new file mode 100644 index 000000000..6283d43dd --- /dev/null +++ b/xmake/actions/build/builtin/prepare_shared.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_shared.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/builtin/prepare_static.lua b/xmake/actions/build/builtin/prepare_static.lua new file mode 100644 index 000000000..ac962d98a --- /dev/null +++ b/xmake/actions/build/builtin/prepare_static.lua @@ -0,0 +1,25 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_static.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) +end diff --git a/xmake/actions/build/deprecated/.DS_Store b/xmake/actions/build/deprecated/.DS_Store deleted file mode 100644 index a8ae5f9d6..000000000 Binary files a/xmake/actions/build/deprecated/.DS_Store and /dev/null differ diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 0c1c64abf..28b5872e4 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -38,8 +38,7 @@ end -- add jobs for the builtin script function _add_jobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then - -- TODO - local script = import("deprecated.kinds." .. target:kind(), {anonymous = true})[job_kind] + local script = import("builtin." .. job_kind .. "_" .. target:kind(), {anonymous = true}) if script then script(jobgraph, target) end -- cgit v1.3.1 From b45bad5035bd1b3af98cd90d9f44c08015958f9a Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 28 Mar 2025 00:43:17 +0800 Subject: prepare sourcefiles --- xmake/actions/build/builtin/prepare_binary.lua | 5 +--- xmake/actions/build/builtin/prepare_moduleonly.lua | 5 +--- xmake/actions/build/builtin/prepare_object.lua | 5 +--- xmake/actions/build/builtin/prepare_shared.lua | 5 +--- .../actions/build/builtin/prepare_sourcefiles.lua | 28 ++++++++++++++++++++++ xmake/actions/build/builtin/prepare_static.lua | 5 +--- 6 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 xmake/actions/build/builtin/prepare_sourcefiles.lua diff --git a/xmake/actions/build/builtin/prepare_binary.lua b/xmake/actions/build/builtin/prepare_binary.lua index 827f02d95..b81abaee3 100644 --- a/xmake/actions/build/builtin/prepare_binary.lua +++ b/xmake/actions/build/builtin/prepare_binary.lua @@ -19,7 +19,4 @@ -- -- imports -import("core.base.option") - -function main(jobgraph, target) -end +inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_moduleonly.lua b/xmake/actions/build/builtin/prepare_moduleonly.lua index 3f133c9ff..4866e016c 100644 --- a/xmake/actions/build/builtin/prepare_moduleonly.lua +++ b/xmake/actions/build/builtin/prepare_moduleonly.lua @@ -19,7 +19,4 @@ -- -- imports -import("core.base.option") - -function main(jobgraph, target) -end +inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_object.lua b/xmake/actions/build/builtin/prepare_object.lua index 7197d9ad5..0bc44b29c 100644 --- a/xmake/actions/build/builtin/prepare_object.lua +++ b/xmake/actions/build/builtin/prepare_object.lua @@ -19,7 +19,4 @@ -- -- imports -import("core.base.option") - -function main(jobgraph, target) -end +inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_shared.lua b/xmake/actions/build/builtin/prepare_shared.lua index 6283d43dd..31a277a53 100644 --- a/xmake/actions/build/builtin/prepare_shared.lua +++ b/xmake/actions/build/builtin/prepare_shared.lua @@ -19,7 +19,4 @@ -- -- imports -import("core.base.option") - -function main(jobgraph, target) -end +inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_sourcefiles.lua b/xmake/actions/build/builtin/prepare_sourcefiles.lua new file mode 100644 index 000000000..1b3414278 --- /dev/null +++ b/xmake/actions/build/builtin/prepare_sourcefiles.lua @@ -0,0 +1,28 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_sourcefiles.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) + jobgraph:add("prepare_sourcefiles", function (index, total, opt) + print("prepare sourcefiles") + end) +end diff --git a/xmake/actions/build/builtin/prepare_static.lua b/xmake/actions/build/builtin/prepare_static.lua index ac962d98a..6427fd1d8 100644 --- a/xmake/actions/build/builtin/prepare_static.lua +++ b/xmake/actions/build/builtin/prepare_static.lua @@ -19,7 +19,4 @@ -- -- imports -import("core.base.option") - -function main(jobgraph, target) -end +inherit("prepare_sourcefiles") -- cgit v1.3.1 From d7473e4d4ceb8e83bd2d6bff5c91373110a08c4d Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 28 Mar 2025 00:44:36 +0800 Subject: improve to prepare files --- xmake/actions/build/builtin/prepare_binary.lua | 22 ----- xmake/actions/build/builtin/prepare_files.lua | 28 ++++++ xmake/actions/build/builtin/prepare_moduleonly.lua | 22 ----- xmake/actions/build/builtin/prepare_object.lua | 22 ----- xmake/actions/build/builtin/prepare_shared.lua | 22 ----- .../actions/build/builtin/prepare_sourcefiles.lua | 28 ------ xmake/actions/build/builtin/prepare_static.lua | 22 ----- xmake/actions/build/target_utils.lua | 109 +++++++++++---------- 8 files changed, 85 insertions(+), 190 deletions(-) delete mode 100644 xmake/actions/build/builtin/prepare_binary.lua create mode 100644 xmake/actions/build/builtin/prepare_files.lua delete mode 100644 xmake/actions/build/builtin/prepare_moduleonly.lua delete mode 100644 xmake/actions/build/builtin/prepare_object.lua delete mode 100644 xmake/actions/build/builtin/prepare_shared.lua delete mode 100644 xmake/actions/build/builtin/prepare_sourcefiles.lua delete mode 100644 xmake/actions/build/builtin/prepare_static.lua diff --git a/xmake/actions/build/builtin/prepare_binary.lua b/xmake/actions/build/builtin/prepare_binary.lua deleted file mode 100644 index b81abaee3..000000000 --- a/xmake/actions/build/builtin/prepare_binary.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_binary.lua --- - --- imports -inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_files.lua b/xmake/actions/build/builtin/prepare_files.lua new file mode 100644 index 000000000..c63257fd0 --- /dev/null +++ b/xmake/actions/build/builtin/prepare_files.lua @@ -0,0 +1,28 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_files.lua +-- + +-- imports +import("core.base.option") + +function main(jobgraph, target) + jobgraph:add("prepare_files", function (index, total, opt) + print("prepare sourcefiles") + end) +end diff --git a/xmake/actions/build/builtin/prepare_moduleonly.lua b/xmake/actions/build/builtin/prepare_moduleonly.lua deleted file mode 100644 index 4866e016c..000000000 --- a/xmake/actions/build/builtin/prepare_moduleonly.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_moduleonly.lua --- - --- imports -inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_object.lua b/xmake/actions/build/builtin/prepare_object.lua deleted file mode 100644 index 0bc44b29c..000000000 --- a/xmake/actions/build/builtin/prepare_object.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_object.lua --- - --- imports -inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_shared.lua b/xmake/actions/build/builtin/prepare_shared.lua deleted file mode 100644 index 31a277a53..000000000 --- a/xmake/actions/build/builtin/prepare_shared.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_shared.lua --- - --- imports -inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/builtin/prepare_sourcefiles.lua b/xmake/actions/build/builtin/prepare_sourcefiles.lua deleted file mode 100644 index 1b3414278..000000000 --- a/xmake/actions/build/builtin/prepare_sourcefiles.lua +++ /dev/null @@ -1,28 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_sourcefiles.lua --- - --- imports -import("core.base.option") - -function main(jobgraph, target) - jobgraph:add("prepare_sourcefiles", function (index, total, opt) - print("prepare sourcefiles") - end) -end diff --git a/xmake/actions/build/builtin/prepare_static.lua b/xmake/actions/build/builtin/prepare_static.lua deleted file mode 100644 index 6427fd1d8..000000000 --- a/xmake/actions/build/builtin/prepare_static.lua +++ /dev/null @@ -1,22 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_static.lua --- - --- imports -inherit("prepare_sourcefiles") diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 28b5872e4..b6f9617c8 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -26,6 +26,7 @@ import("core.project.project") import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") +import("builtin.prepare_files") -- clean target for rebuilding function _clean_target(target) @@ -38,60 +39,15 @@ end -- add jobs for the builtin script function _add_jobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then - local script = import("builtin." .. job_kind .. "_" .. target:kind(), {anonymous = true}) - if script then - script(jobgraph, target) - end - end -end - --- add jobs for the given script -function _add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) - local has_script = false - local script = instance:script(script_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build(function (target, jobgraph, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then - script(target, jobgraph) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + if job_kind == "prepare" then + prepare_files(jobgraph, target) else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = opt.progress}) - end) - end - has_script = true - else - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, opt) - -- end) - local scriptcmd = instance:script(scriptcmd_name) - if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true + local script = import("builtin.build_" .. target:kind(), {anonymous = true}) + if script then + script(jobgraph, target) + end end end - return has_script end -- add jobs for the given target stage @@ -120,7 +76,7 @@ function _add_jobs_for_target_stage(jobgraph, target, stage, opt) jobgraph:group(group_name, function () local has_script = false for _, instance in ipairs(instances) do - if _add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then + if add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then has_script = true end end @@ -220,6 +176,55 @@ function _get_jobs(targets_root, opt) return jobgraph end +-- add jobs for the given script +function add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) + local has_script = false + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true + else + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + -- get all root targets function get_root_targets(targetnames, opt) opt = opt or {} -- cgit v1.3.1 From e7a426b2e6b9e9657d7e6f5eb264959218ced846 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:32:11 +0800 Subject: pass distcc --- xmake/actions/build/build.lua | 7 ++++--- xmake/actions/build/target_utils.lua | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 63121ea07..824e8c5c5 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -31,12 +31,13 @@ function _prepare(targets_root, opt) target_utils.runjobs(targets_root, opt) end --- TODO distcc function _build(targets_root, opt) - --[[ opt = opt or {} opt.job_kind = "build" - target_utils.runjobs(targets_root, opt)]] + if distcc_build_client.is_connected() then + opt.distcc = distcc_build_client.singleton() + end + target_utils.runjobs(targets_root, opt) end function main(targetnames, opt) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index b6f9617c8..d411ed53e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -284,7 +284,7 @@ function runjobs(targets_root, opt) if errors and progress.showing_without_scroll() then print("") end - end, comax = option.get("jobs") or 1, curdir = curdir}) + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) os.cd(curdir) end end -- cgit v1.3.1 From b8949110acd9d2b76ff5c214d95fdc4dda076b5a Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:33:56 +0800 Subject: rename targetjobs --- xmake/actions/build/build.lua | 4 +- xmake/actions/build/target_utils.lua | 141 ++++++++++++++++++----------------- 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 824e8c5c5..6a4c14ed5 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -28,7 +28,7 @@ import("deprecated.build", {alias = "deprecated_build"}) function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" - target_utils.runjobs(targets_root, opt) + target_utils.run_targetjobs(targets_root, opt) end function _build(targets_root, opt) @@ -37,7 +37,7 @@ function _build(targets_root, opt) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end - target_utils.runjobs(targets_root, opt) + target_utils.run_targetjobs(targets_root, opt) end function main(targetnames, opt) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index d411ed53e..613063ef2 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -36,8 +36,8 @@ function _clean_target(target) end end --- add jobs for the builtin script -function _add_jobs_for_builtin_script(jobgraph, target, job_kind) +-- add target jobs for the builtin script +function _add_targetjobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then if job_kind == "prepare" then prepare_files(jobgraph, target) @@ -50,9 +50,58 @@ function _add_jobs_for_builtin_script(jobgraph, target, job_kind) end end --- add jobs for the given target stage +-- add target jobs for the given script +function _add_targetjobs_for_script(jobgraph, instance, script_name, scriptcmd_name) + local has_script = false + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true + else + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add target jobs with the given stage -- stage: before, after or "" -function _add_jobs_for_target_stage(jobgraph, target, stage, opt) +function _add_targetjobs_with_stage(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind @@ -76,14 +125,14 @@ function _add_jobs_for_target_stage(jobgraph, target, stage, opt) jobgraph:group(group_name, function () local has_script = false for _, instance in ipairs(instances) do - if add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then + if _add_targetjobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then has_script = true end end -- call builtin script, e.g. on_prepare, on_build, ... if not has_script and stage == "" then - _add_jobs_for_builtin_script(jobgraph, target, job_kind) + _add_targetjobs_for_builtin_script(jobgraph, target, job_kind) end end) @@ -92,8 +141,8 @@ function _add_jobs_for_target_stage(jobgraph, target, stage, opt) end end --- add jobs for the given target -function _add_jobs_for_target(jobgraph, target, opt) +-- add target jobs for the given target +function _add_targetjobs(jobgraph, target, opt) opt = opt or {} if not target:is_enabled() then return @@ -146,85 +195,36 @@ function _add_jobs_for_target(jobgraph, target, opt) end end) - -- add jobs for target stage, e.g. after_xxx -> (depend on) on_xxx -> before_xxx - local group = _add_jobs_for_target_stage(jobgraph, target, "", opt) - local group_before = _add_jobs_for_target_stage(jobgraph, target, "before", opt) - local group_after = _add_jobs_for_target_stage(jobgraph, target, "after", opt) + -- add jobs with target stage, e.g. after_xxx -> (depend on) on_xxx -> before_xxx + local group = _add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = _add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = _add_targetjobs_with_stage(jobgraph, target, "after", opt) jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end --- add jobs for the given target and deps -function _add_jobs_for_target_and_deps(jobgraph, target, targetrefs, opt) +-- add target jobs for the given target and deps +function _add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target - _add_jobs_for_target(jobgraph, target, opt) + _add_targetjobs(jobgraph, target, opt) for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) - _add_jobs_for_target_and_deps(jobgraph, dep, targetrefs, opt) + _add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) end end end --- get jobs -function _get_jobs(targets_root, opt) +-- get target jobs +function _get_targetjobs(targets_root, opt) local jobgraph = async_jobgraph.new() local targetrefs = {} for _, target in ipairs(targets_root) do - _add_jobs_for_target_and_deps(jobgraph, target, targetrefs, opt) + _add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) end return jobgraph end --- add jobs for the given script -function add_jobs_for_script(jobgraph, instance, script_name, scriptcmd_name) - local has_script = false - local script = instance:script(script_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build(function (target, jobgraph, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then - script(target, jobgraph) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = opt.progress}) - end) - end - has_script = true - else - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, opt) - -- end) - local scriptcmd = instance:script(scriptcmd_name) - if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - return has_script -end - -- get all root targets function get_root_targets(targetnames, opt) opt = opt or {} @@ -273,10 +273,11 @@ function get_root_targets(targetnames, opt) return targets_root end -function runjobs(targets_root, opt) +-- run target-level jobs, e.g. on_prepare, on_build, ... +function run_targetjobs(targets_root, opt) opt = opt or {} local job_kind = opt.job_kind - local jobgraph = _get_jobs(targets_root, opt) + local jobgraph = _get_targetjobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() async_runjobs(job_kind, jobgraph, {on_exit = function (errors) -- cgit v1.3.1 From 6be0431dd8d7462d53fc4cce2609adfacc57089a Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:37:12 +0800 Subject: add filejobs stub --- xmake/actions/build/build.lua | 3 ++ xmake/actions/build/build_files.lua | 60 ++++++++++++++++++++++++++++++++ xmake/actions/build/deprecated/build.lua | 1 - xmake/actions/build/target_utils.lua | 50 ++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 6a4c14ed5..22097669a 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -23,14 +23,17 @@ import("core.base.option") import("core.project.config") import("core.project.project") import("target_utils") +import("private.service.distcc_build.client", {alias = "distcc_build_client"}) import("deprecated.build", {alias = "deprecated_build"}) +-- run prepare jobs function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" target_utils.run_targetjobs(targets_root, opt) end +-- run build jobs function _build(targets_root, opt) opt = opt or {} opt.job_kind = "build" diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 718390e0b..e26c11aae 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -24,12 +24,72 @@ import("core.base.hashset") import("core.project.config") import("core.project.project") import("target_utils") +import("private.service.distcc_build.client", {alias = "distcc_build_client"}) import("deprecated.build_files", {alias = "deprecated_build_files"}) +-- convert all sourcefiles to lua pattern +function _get_file_patterns(sourcefiles) + local patterns = {} + for _, sourcefile in ipairs(path.splitenv(sourcefiles)) do + + -- get the excludes + local pattern = sourcefile:trim() + local excludes = pattern:match("|.*$") + if excludes then excludes = excludes:split("|", {plain = true}) end + + -- translate excludes + if excludes then + local _excludes = {} + for _, exclude in ipairs(excludes) do + exclude = path.translate(exclude) + exclude = path.pattern(exclude) + table.insert(_excludes, exclude) + end + excludes = _excludes + end + + -- translate path and remove some repeat separators + pattern = path.translate(pattern:gsub("|.*$", "")) + + -- remove "./" or '.\\' prefix + if pattern:sub(1, 2):find('%.[/\\]') then + pattern = pattern:sub(3) + end + + -- get the root directory + local rootdir = pattern + local startpos = pattern:find("*", 1, true) + if startpos then + rootdir = rootdir:sub(1, startpos - 1) + end + rootdir = path.directory(rootdir) + + -- convert to lua path pattern + pattern = path.pattern(pattern) + table.insert(patterns, {pattern = pattern, excludes = excludes, rootdir = rootdir}) + end + return patterns +end + +-- run prepare files jobs function _prepare_files(targets_root, opt) + opt = opt or {} + opt.job_kind = "prepare" + opt.filepatterns = _get_file_patterns(opt.sourcefiles) + target_utils.run_filejobs(targets_root, opt) end +-- run build files jobs function _build_files(targets_root, opt) + opt = opt or {} + opt.job_kind = "build" + opt.filepatterns = _get_file_patterns(opt.sourcefiles) + if distcc_build_client.is_connected() then + opt.distcc = distcc_build_client.singleton() + end + if not target_utils.run_filejobs(targets_root, opt) then + wprint("%s not found!", opt.sourcefiles) + end end function main(targetnames, opt) diff --git a/xmake/actions/build/deprecated/build.lua b/xmake/actions/build/deprecated/build.lua index 1c8d47a18..377ab5e18 100644 --- a/xmake/actions/build/deprecated/build.lua +++ b/xmake/actions/build/deprecated/build.lua @@ -26,7 +26,6 @@ import("private.async.jobpool") import("async.runjobs") import("private.utils.batchcmds") import("core.base.hashset") -import("private.service.remote_cache.client", {alias = "remote_cache_client"}) import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -- clean target for rebuilding diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 613063ef2..7708bba88 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -225,6 +225,37 @@ function _get_targetjobs(targets_root, opt) return jobgraph end +-- add file jobs for the given target +function _add_filejobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end +end + +-- add file jobs for the given target and deps +function _add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + _add_filejobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + _add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) + end + end +end + +-- get files jobs +function _get_filejobs(targets_root, opt) + local jobgraph = async_jobgraph.new() + local targetrefs = {} + for _, target in ipairs(targets_root) do + _add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + -- get all root targets function get_root_targets(targetnames, opt) opt = opt or {} @@ -287,5 +318,24 @@ function run_targetjobs(targets_root, opt) end end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) os.cd(curdir) + return true + end +end + +-- run files-level jobs, e.g. on_prepare_files, on_build_files, ... +function run_filejobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = _get_filejobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) + os.cd(curdir) + return true end end -- cgit v1.3.1 From fec05b2bbe38fcaa6b6eb1546fad2f4ab80300b1 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:39:19 +0800 Subject: match sourcebatches --- xmake/actions/build/target_utils.lua | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 7708bba88..3563bdc2c 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -225,12 +225,72 @@ function _get_targetjobs(targets_root, opt) return jobgraph end +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- match sourcebatches +function _match_sourcebatches(target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return newbatches + end +end + -- add file jobs for the given target function _add_filejobs(jobgraph, target, opt) opt = opt or {} if not target:is_enabled() then return end + + -- get sourcebatches + local filepatterns = opt.filepatterns + local sourcebatches = filepatterns and _match_sourcebatches(target, filepatterns) or target:sourcebatches() + end -- add file jobs for the given target and deps -- cgit v1.3.1 From 2cf6a82530ff9bc8fea4730fe5458ca3e60470df Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:50:47 +0800 Subject: do prepare files --- xmake/actions/build/target_utils.lua | 155 ++++++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3563bdc2c..ae141027d 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -21,6 +21,7 @@ -- imports import("core.base.option") import("core.base.hashset") +import("core.project.rule") import("core.project.config") import("core.project.project") import("async.runjobs", {alias = "async_runjobs"}) @@ -28,6 +29,14 @@ import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") import("builtin.prepare_files") +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function _get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + -- clean target for rebuilding function _clean_target(target) if target:targetfile() then @@ -195,7 +204,7 @@ function _add_targetjobs(jobgraph, target, opt) end end) - -- add jobs with target stage, e.g. after_xxx -> (depend on) on_xxx -> before_xxx + -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx local group = _add_targetjobs_with_stage(jobgraph, target, "", opt) local group_before = _add_targetjobs_with_stage(jobgraph, target, "before", opt) local group_after = _add_targetjobs_with_stage(jobgraph, target, "after", opt) @@ -280,6 +289,122 @@ function _match_sourcebatches(target, filepatterns) end end +-- add file jobs for the builtin script +function _add_filejobs_for_builtin_script(jobgraph, target, sourcebatch, job_kind) +end + +-- add file jobs for the given script, TODO on single file +function _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) + local has_script = false + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, jobgraph, sourcebatch, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph, sourcebatch) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, sourcebatch, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, sourcebatch, {progress = opt.progress}) + end) + end + has_script = true + else + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, sourcebatch, opt) + -- end) + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, sourcebatch, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add file jobs with the given stage +-- stage: before, after or "" +-- +function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + local job_kind_files = job_kind .. "_files" + + -- the group name, e.g. foo/after_prepare_files, bar/before_build_files + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage, job_kind) + + -- the script name, e.g. before/after_prepare_files, before/after_build_files + local script_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files + + -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files + local scriptcmd_name = stage ~= "" and (job_kind_files .. "cmd_" .. stage) or (job_kind_files .. "cmd") + + -- build sourcebatches map + local sourcebatches_map = {} + for _, sourcebatch in ipairs(sourcebatches) do + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local ruleinst = _get_rule(target, rulename) + sourcebatches_map[ruleinst] = sourcebatch + end + + -- TODO sort rules and jobs + local instances = {target} + for _, r in ipairs(target:orderules()) do + table.insert(instances, r) + end + + -- call target and rules script + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local has_script = false + for _, instance in ipairs(instances) do + if instance == target then + for _, sourcebatch in ipairs(sourcebatches) do + if _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) then + has_script = true + end + end + else -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch and _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) then + has_script = true + end + end + end + + -- call builtin script, e.g. on_prepare_files, on_build_files, ... + if not has_script and stage == "" then + for _, sourcebatch in ipairs(sourcebatches) do + _add_filejobs_for_builtin_script(jobgraph, target, sourcebatch, job_kind) + end + end + end) + + if jobgraph:size() > jobsize then + return group_name + end +end + -- add file jobs for the given target function _add_filejobs(jobgraph, target, opt) opt = opt or {} @@ -291,6 +416,34 @@ function _add_filejobs(jobgraph, target, opt) local filepatterns = opt.filepatterns local sourcebatches = filepatterns and _match_sourcebatches(target, filepatterns) or target:sourcebatches() + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + local sourcebatches_result = {} + for _, sourcebatch in pairs(sourcebatches) do + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = _get_rule(target, rulename) + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_result, sourcebatch) + end + else + table.insert(sourcebatches_result, sourcebatch) + end + end + if #sourcebatches_result == 0 then + return + end + + -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files + local group = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "", opt) + local group_before = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "before", opt) + local group_after = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "after", opt) + jobgraph:add_orders(group_before, group, group_after) end -- add file jobs for the given target and deps -- cgit v1.3.1 From e1891c905e406f16738e918df3a84b9acbcd94dd Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:53:39 +0800 Subject: improve script opt --- xmake/actions/build/target_utils.lua | 50 ++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index ae141027d..bd9767a82 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -60,8 +60,10 @@ function _add_targetjobs_for_builtin_script(jobgraph, target, job_kind) end -- add target jobs for the given script -function _add_targetjobs_for_script(jobgraph, instance, script_name, scriptcmd_name) +function _add_targetjobs_for_script(jobgraph, instance, opt) + opt = opt or {} local has_script = false + local script_name = opt.script_name local script = instance:script(script_name) if script then -- call custom script with jobgraph @@ -94,6 +96,7 @@ function _add_targetjobs_for_script(jobgraph, instance, script_name, scriptcmd_n -- target("test") -- on_buildcmd(function (target, batchcmds, opt) -- end) + local scriptcmd_name = opt.scriptcmd_name local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) @@ -133,8 +136,12 @@ function _add_targetjobs_with_stage(jobgraph, target, stage, opt) local jobsize = jobgraph:size() jobgraph:group(group_name, function () local has_script = false + local script_opt = { + script_name = script_name, + scriptcmd_name = scriptcmd_name + } for _, instance in ipairs(instances) do - if _add_targetjobs_for_script(jobgraph, instance, script_name, scriptcmd_name) then + if _add_targetjobs_for_script(jobgraph, instance, script_opt) then has_script = true end end @@ -294,9 +301,12 @@ function _add_filejobs_for_builtin_script(jobgraph, target, sourcebatch, job_kin end -- add file jobs for the given script, TODO on single file -function _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) +function _add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) + opt = opt or {} local has_script = false - local script = instance:script(script_name) + local script_file_name = opt.script_file_name + local script_files_name = opt.script_files_name + local script = instance:script(script_files_name) if script then -- call custom script with jobgraph -- e.g. @@ -304,10 +314,11 @@ function _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, -- target("test") -- on_build_files(function (target, jobgraph, sourcebatch, opt) -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then + if instance:extraconf(script_files_name, "jobgraph") then script(target, jobgraph, sourcebatch) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + elseif instance:extraconf(script_files_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_files_name) else -- call custom script directly -- e.g. @@ -315,7 +326,7 @@ function _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, -- target("test") -- on_build_files(function (target, sourcebatch, opt) -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_files_name) jobgraph:add(jobname, function (index, total, opt) script(target, sourcebatch, {progress = opt.progress}) end) @@ -328,9 +339,11 @@ function _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, -- target("test") -- on_buildcmd(function (target, batchcmds, sourcebatch, opt) -- end) - local scriptcmd = instance:script(scriptcmd_name) + local scriptcmd_file_name = opt.scriptcmd_file_name + local scriptcmd_files_name = opt.scriptcmd_files_name + local scriptcmd = instance:script(scriptcmd_files_name) if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, sourcebatch, {progress = opt.progress}) @@ -348,16 +361,19 @@ end function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} local job_kind = opt.job_kind + local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" -- the group name, e.g. foo/after_prepare_files, bar/before_build_files local group_name = string.format("%s/%s_%s_files", target:fullname(), stage, job_kind) -- the script name, e.g. before/after_prepare_files, before/after_build_files - local script_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files + local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file + local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files - local scriptcmd_name = stage ~= "" and (job_kind_files .. "cmd_" .. stage) or (job_kind_files .. "cmd") + local scriptcmd_file_name = stage ~= "" and (job_kind_file .. "cmd_" .. stage) or (job_kind_file .. "cmd") + local scriptcmd_files_name = stage ~= "" and (job_kind_files .. "cmd_" .. stage) or (job_kind_files .. "cmd") -- build sourcebatches map local sourcebatches_map = {} @@ -377,16 +393,22 @@ function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local jobsize = jobgraph:size() jobgraph:group(group_name, function () local has_script = false + local script_opt = { + script_file_name = script_file_name, + script_files_name = script_files_name, + scriptcmd_file_name = scriptcmd_file_name, + scriptcmd_files_name = scriptcmd_files_name + } for _, instance in ipairs(instances) do if instance == target then for _, sourcebatch in ipairs(sourcebatches) do - if _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) then + if _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) then has_script = true end end else -- rule local sourcebatch = sourcebatches_map[instance] - if sourcebatch and _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_name, scriptcmd_name) then + if sourcebatch and _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) then has_script = true end end -- cgit v1.3.1 From 6e967051effcbb383a1c8925857625424d384ee8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:56:30 +0800 Subject: add prepare_file --- xmake/actions/build/target_utils.lua | 201 ++++++++++++++++++++++++----------- 1 file changed, 138 insertions(+), 63 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index bd9767a82..e3c700748 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -63,39 +63,45 @@ end function _add_targetjobs_for_script(jobgraph, instance, opt) opt = opt or {} local has_script = false - local script_name = opt.script_name - local script = instance:script(script_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build(function (target, jobgraph, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then - script(target, jobgraph) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) - else - -- call custom script directly + + -- call script + if not has_script then + local script_name = opt.script_name + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph -- e.g. -- -- target("test") - -- on_build(function (target, opt) - -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = opt.progress}) - end) + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true end - has_script = true - else - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, opt) - -- end) + end + + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + if not has_script then local scriptcmd_name = opt.scriptcmd_name local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then @@ -304,49 +310,118 @@ end function _add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) opt = opt or {} local has_script = false - local script_file_name = opt.script_file_name - local script_files_name = opt.script_files_name - local script = instance:script(script_files_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build_files(function (target, jobgraph, sourcebatch, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_files_name, "jobgraph") then - script(target, jobgraph, sourcebatch) - elseif instance:extraconf(script_files_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", - instance:fullname(), script_files_name) - else - -- call custom script directly + + -- call script files + if not has_script then + local script_files_name = opt.script_files_name + local script_files = instance:script(script_files_name) + if script_files then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, jobgraph, sourcebatch, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_files_name, "distcc") + if instance:extraconf(script_files_name, "jobgraph") then + script_files(target, jobgraph, sourcebatch, {distcc = distcc}) + elseif instance:extraconf(script_files_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_files_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, sourcebatch, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_files_name) + jobgraph:add(jobname, function (index, total, opt) + script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) + end) + end + has_script = true + end + end + + -- call script file + if not has_script then + local script_file_name = opt.script_file_name + local script_file = instance:script(script_file_name) + if script_file then + -- call custom script with jobgraph -- e.g. -- -- target("test") - -- on_build_files(function (target, sourcebatch, opt) - -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_files_name) + -- on_build_file(function (target, jobgraph, sourcefile, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_file_name, "distcc") + if instance:extraconf(script_file_name, "jobgraph") then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) + end + elseif instance:extraconf(script_file_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_file_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, sourcefile, opt) + -- end) + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_file_name) + jobgraph:add(jobname, function (index, total, opt) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + end) + end + has_script = true + end + end + + -- call command script files + -- e.g. + -- + -- target("test") + -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- end) + if not has_script then + local scriptcmd_files_name = opt.scriptcmd_files_name + local scriptcmd_files = instance:script(scriptcmd_files_name) + if scriptcmd_files then + local distcc = instance:extraconf(scriptcmd_files_name, "distcc") + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) - script(target, sourcebatch, {progress = opt.progress}) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end) + has_script = true end - has_script = true - else - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, sourcebatch, opt) - -- end) + end + + -- call command script file + -- e.g. + -- + -- target("test") + -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- end) + if not has_script then local scriptcmd_file_name = opt.scriptcmd_file_name - local scriptcmd_files_name = opt.scriptcmd_files_name - local scriptcmd = instance:script(scriptcmd_files_name) - if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_files_name) + local scriptcmd_file = instance:script(scriptcmd_file_name) + if scriptcmd_file then + local distcc = instance:extraconf(scriptcmd_file_name, "distcc") + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_file_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, sourcebatch, {progress = opt.progress}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end) has_script = true -- cgit v1.3.1 From ec6334f3c829925ecddb36c36d3ae878db18ef4e Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:58:28 +0800 Subject: update comment --- xmake/actions/build/target_utils.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index e3c700748..a38095808 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -408,7 +408,7 @@ function _add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) -- e.g. -- -- target("test") - -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) -- end) if not has_script then local scriptcmd_file_name = opt.scriptcmd_file_name -- cgit v1.3.1 From e4867e248c7ab82df6e8daafaa9a66bb84320db5 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:59:24 +0800 Subject: remove builtin files --- xmake/actions/build/target_utils.lua | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index a38095808..5aa368a14 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -302,10 +302,6 @@ function _match_sourcebatches(target, filepatterns) end end --- add file jobs for the builtin script -function _add_filejobs_for_builtin_script(jobgraph, target, sourcebatch, job_kind) -end - -- add file jobs for the given script, TODO on single file function _add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) opt = opt or {} @@ -467,7 +463,6 @@ function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) -- call target and rules script local jobsize = jobgraph:size() jobgraph:group(group_name, function () - local has_script = false local script_opt = { script_file_name = script_file_name, script_files_name = script_files_name, @@ -477,24 +472,15 @@ function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, instance in ipairs(instances) do if instance == target then for _, sourcebatch in ipairs(sourcebatches) do - if _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) then - has_script = true - end + _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) end else -- rule local sourcebatch = sourcebatches_map[instance] - if sourcebatch and _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) then - has_script = true + if sourcebatch then + _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) end end end - - -- call builtin script, e.g. on_prepare_files, on_build_files, ... - if not has_script and stage == "" then - for _, sourcebatch in ipairs(sourcebatches) do - _add_filejobs_for_builtin_script(jobgraph, target, sourcebatch, job_kind) - end - end end) if jobgraph:size() > jobsize then -- cgit v1.3.1 From 386cf48145dcf2f21e003ccb71b36f1de169e169 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 00:54:50 +0800 Subject: prepare files --- xmake/actions/build/builtin/prepare_files.lua | 5 +- xmake/actions/build/target_utils.lua | 176 +++++++++++++------------- 2 files changed, 88 insertions(+), 93 deletions(-) diff --git a/xmake/actions/build/builtin/prepare_files.lua b/xmake/actions/build/builtin/prepare_files.lua index c63257fd0..320d8cebb 100644 --- a/xmake/actions/build/builtin/prepare_files.lua +++ b/xmake/actions/build/builtin/prepare_files.lua @@ -20,9 +20,8 @@ -- imports import("core.base.option") +import(".target_utils") function main(jobgraph, target) - jobgraph:add("prepare_files", function (index, total, opt) - print("prepare sourcefiles") - end) + target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 5aa368a14..49066f23a 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,7 +27,6 @@ import("core.project.project") import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") -import("builtin.prepare_files") -- get rule -- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target @@ -45,22 +44,74 @@ function _clean_target(target) end end +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- match sourcebatches +function _match_sourcebatches(target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return newbatches + end +end + -- add target jobs for the builtin script -function _add_targetjobs_for_builtin_script(jobgraph, target, job_kind) +function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then if job_kind == "prepare" then - prepare_files(jobgraph, target) + import("builtin.prepare_files", {anonymous = true})(jobgraph, target) else - local script = import("builtin.build_" .. target:kind(), {anonymous = true}) - if script then - script(jobgraph, target) - end + import("builtin.build_" .. target:kind(), {anonymous = true})(jobgraph, target) end end end -- add target jobs for the given script -function _add_targetjobs_for_script(jobgraph, instance, opt) +function add_targetjobs_for_script(jobgraph, instance, opt) opt = opt or {} local has_script = false @@ -119,7 +170,7 @@ end -- add target jobs with the given stage -- stage: before, after or "" -function _add_targetjobs_with_stage(jobgraph, target, stage, opt) +function add_targetjobs_with_stage(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind @@ -147,14 +198,14 @@ function _add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do - if _add_targetjobs_for_script(jobgraph, instance, script_opt) then + if add_targetjobs_for_script(jobgraph, instance, script_opt) then has_script = true end end -- call builtin script, e.g. on_prepare, on_build, ... if not has_script and stage == "" then - _add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + add_targetjobs_for_builtin_script(jobgraph, target, job_kind) end end) @@ -164,7 +215,7 @@ function _add_targetjobs_with_stage(jobgraph, target, stage, opt) end -- add target jobs for the given target -function _add_targetjobs(jobgraph, target, opt) +function add_targetjobs(jobgraph, target, opt) opt = opt or {} if not target:is_enabled() then return @@ -218,92 +269,37 @@ function _add_targetjobs(jobgraph, target, opt) end) -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx - local group = _add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = _add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = _add_targetjobs_with_stage(jobgraph, target, "after", opt) + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end -- add target jobs for the given target and deps -function _add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) +function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target - _add_targetjobs(jobgraph, target, opt) + add_targetjobs(jobgraph, target, opt) for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) - _add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) end end end -- get target jobs -function _get_targetjobs(targets_root, opt) +function get_targetjobs(targets_root, opt) local jobgraph = async_jobgraph.new() local targetrefs = {} for _, target in ipairs(targets_root) do - _add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) end return jobgraph end --- match source files -function _match_sourcefiles(sourcefile, filepatterns) - for _, filepattern in ipairs(filepatterns) do - if sourcefile:match(filepattern.pattern) == sourcefile then - if filepattern.excludes then - if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then - sourcefile = sourcefile:sub(#filepattern.rootdir + 2) - end - for _, exclude in ipairs(filepattern.excludes) do - if sourcefile:match(exclude) == sourcefile then - return false - end - end - end - return true - end - end -end - --- match sourcebatches -function _match_sourcebatches(target, filepatterns) - local newbatches = {} - local sourcecount = 0 - for rulename, sourcebatch in pairs(target:sourcebatches()) do - local objectfiles = sourcebatch.objectfiles - local dependfiles = sourcebatch.dependfiles - local sourcekind = sourcebatch.sourcekind - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do - if _match_sourcefiles(sourcefile, filepatterns) then - local newbatch = newbatches[rulename] - if not newbatch then - newbatch = {} - newbatch.sourcekind = sourcekind - newbatch.rulename = rulename - newbatch.sourcefiles = {} - end - table.insert(newbatch.sourcefiles, sourcefile) - if objectfiles then - newbatch.objectfiles = newbatch.objectfiles or {} - table.insert(newbatch.objectfiles, objectfiles[idx]) - end - if dependfiles then - newbatch.dependfiles = newbatch.dependfiles or {} - table.insert(newbatch.dependfiles, dependfiles[idx]) - end - newbatches[rulename] = newbatch - sourcecount = sourcecount + 1 - end - end - end - if sourcecount > 0 then - return newbatches - end -end - -- add file jobs for the given script, TODO on single file -function _add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) +function add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) opt = opt or {} local has_script = false @@ -429,7 +425,7 @@ end -- add file jobs with the given stage -- stage: before, after or "" -- -function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) +function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" @@ -472,12 +468,12 @@ function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, instance in ipairs(instances) do if instance == target then for _, sourcebatch in ipairs(sourcebatches) do - _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) + add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) end else -- rule local sourcebatch = sourcebatches_map[instance] if sourcebatch then - _add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) + add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) end end end @@ -489,7 +485,7 @@ function _add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) end -- add file jobs for the given target -function _add_filejobs(jobgraph, target, opt) +function add_filejobs(jobgraph, target, opt) opt = opt or {} if not target:is_enabled() then return @@ -523,31 +519,31 @@ function _add_filejobs(jobgraph, target, opt) end -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "", opt) - local group_before = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "before", opt) - local group_after = _add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "after", opt) + local group = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "", opt) + local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "before", opt) + local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "after", opt) jobgraph:add_orders(group_before, group, group_after) end -- add file jobs for the given target and deps -function _add_filejobs_and_deps(jobgraph, target, targetrefs, opt) +function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target - _add_filejobs(jobgraph, target, opt) + add_filejobs(jobgraph, target, opt) for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) - _add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) + add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) end end end -- get files jobs -function _get_filejobs(targets_root, opt) +function get_filejobs(targets_root, opt) local jobgraph = async_jobgraph.new() local targetrefs = {} for _, target in ipairs(targets_root) do - _add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + add_filejobs_and_deps(jobgraph, target, targetrefs, opt) end return jobgraph end @@ -604,7 +600,7 @@ end function run_targetjobs(targets_root, opt) opt = opt or {} local job_kind = opt.job_kind - local jobgraph = _get_targetjobs(targets_root, opt) + local jobgraph = get_targetjobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() async_runjobs(job_kind, jobgraph, {on_exit = function (errors) @@ -622,7 +618,7 @@ end function run_filejobs(targets_root, opt) opt = opt or {} local job_kind = opt.job_kind - local jobgraph = _get_filejobs(targets_root, opt) + local jobgraph = get_filejobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() async_runjobs(job_kind, jobgraph, {on_exit = function (errors) -- cgit v1.3.1 From a10183066026e54be1aabc1750a9dd41bc447aa0 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 01:01:12 +0800 Subject: fix nil target --- xmake/actions/build/target_utils.lua | 10 +++++----- xmake/core/project/policy.lua | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 49066f23a..dfb21b23f 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -111,7 +111,7 @@ function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) end -- add target jobs for the given script -function add_targetjobs_for_script(jobgraph, instance, opt) +function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false @@ -198,7 +198,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do - if add_targetjobs_for_script(jobgraph, instance, script_opt) then + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then has_script = true end end @@ -299,7 +299,7 @@ function get_targetjobs(targets_root, opt) end -- add file jobs for the given script, TODO on single file -function add_filejobs_for_script(jobgraph, instance, sourcebatch, opt) +function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) opt = opt or {} local has_script = false @@ -468,12 +468,12 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, instance in ipairs(instances) do if instance == target then for _, sourcebatch in ipairs(sourcebatches) do - add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) end else -- rule local sourcebatch = sourcebatches_map[instance] if sourcebatch then - add_filejobs_for_script(jobgraph, instance, sourcebatch, script_opt) + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) end end end diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 76016e164..aae8e83df 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -95,7 +95,7 @@ function policy.policies() -- Enable cuda device link ["build.cuda.devlink"] = {description = "Enable Cuda devlink.", type = "boolean"}, -- Enable build jobgraph - ["build.jobgraph"] = {description = "Enable build jobgraph.", default = false, type = "boolean"}, + ["build.jobgraph"] = {description = "Enable build jobgraph.", default = true, type = "boolean"}, -- Enable windows UAC and set level, e.g. invoker, admin, highest ["windows.manifest.uac"] = {description = "Enable windows manifest UAC.", type = "string"}, -- Enable ui access for windows UAC -- cgit v1.3.1 From b8b1f389c02bdb5521699e73622ea674d676fa90 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 01:05:01 +0800 Subject: build binary stub --- xmake/actions/build/builtin/build_binary.lua | 3 +++ xmake/actions/build/builtin/build_object.lua | 2 ++ 2 files changed, 5 insertions(+) diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua index 591c679c3..89c24a1e7 100644 --- a/xmake/actions/build/builtin/build_binary.lua +++ b/xmake/actions/build/builtin/build_binary.lua @@ -20,6 +20,9 @@ -- imports import("core.base.option") +import("build_object") function main(jobgraph, target) + build_object(jobgraph, target) + -- TODO add link jobs end diff --git a/xmake/actions/build/builtin/build_object.lua b/xmake/actions/build/builtin/build_object.lua index 0d13f392e..f12dce686 100644 --- a/xmake/actions/build/builtin/build_object.lua +++ b/xmake/actions/build/builtin/build_object.lua @@ -20,6 +20,8 @@ -- imports import("core.base.option") +import(".target_utils") function main(jobgraph, target) + target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) end -- cgit v1.3.1 From 4b47fa99489e59337b056e0c95b50493c050fe32 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:24:46 +0800 Subject: improve to build object to support jobgraph --- xmake/modules/private/action/build/object.lua | 27 +++++++++++++++++++++++++-- xmake/rules/c++/xmake.lua | 4 ++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index ae6bf9f4a..8f71f4b68 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -143,8 +143,8 @@ function build(target, sourcebatch, opt) end end --- add batch jobs to build the source files -function main(target, batchjobs, sourcebatch, opt) +-- add build jobs to batchjobs +function _add_batchjobs(target, batchjobs, sourcebatch, opt) local rootjob = opt.rootjob for i = 1, #sourcebatch.sourcefiles do local sourcefile = sourcebatch.sourcefiles[i] @@ -157,3 +157,26 @@ function main(target, batchjobs, sourcebatch, opt) end, {rootjob = rootjob, distcc = opt.distcc}) end end + +-- add build jobs to jobgraph +function _add_jobgraph(target, jobgraph, sourcebatch, opt) + for i = 1, #sourcebatch.sourcefiles do + local sourcefile = sourcebatch.sourcefiles[i] + local objectfile = sourcebatch.objectfiles[i] + local dependfile = sourcebatch.dependfiles[i] + local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) + local jobname = target:fullname() .. "/" .. sourcefile + jobgraph:add(jobname, function (index, total, jobopt) + local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) + build_object(target, sourcefile, build_opt) + end, {distcc = opt.distcc}) + end +end + +function main(target, jobs, sourcebatch, opt) + if jobs.add_orders then + _add_jobgraph(target, jobs, sourcebatch, opt) + else + _add_batchjobs(target, jobs, sourcebatch, opt) + end +end diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua index 94bba431e..abe6cf334 100644 --- a/xmake/rules/c++/xmake.lua +++ b/xmake/rules/c++/xmake.lua @@ -21,7 +21,7 @@ rule("c.build") set_sourcekinds("cc") add_deps("c.build.pcheader", "c.build.optimization", "c.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) -- enable vs runtime as MD by default if target:is_plat("windows") and not target:get("runtimes") then @@ -43,7 +43,7 @@ rule("c.build") rule("c++.build") set_sourcekinds("cxx") add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization", "c++.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) -- enable c++ exceptions by default if target:is_plat("windows") and not target:get("exceptions") then -- cgit v1.3.1 From 92d6126d1c98c9c58d38f5b173d7248a1e882d20 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:32:11 +0800 Subject: enable more build.object to jobgraph --- xmake/modules/private/action/build/object.lua | 8 ++++---- xmake/rules/asm/xmake.lua | 2 +- xmake/rules/cuda/xmake.lua | 2 +- xmake/rules/dlang/xmake.lua | 2 +- xmake/rules/objc++/xmake.lua | 4 ++-- xmake/rules/swift/xmake.lua | 2 +- xmake/rules/winsdk/xmake.lua | 2 +- xmake/rules/zig/xmake.lua | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 8f71f4b68..5118dbb82 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -173,10 +173,10 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) end end -function main(target, jobs, sourcebatch, opt) - if jobs.add_orders then - _add_jobgraph(target, jobs, sourcebatch, opt) +function main(target, jobgraph, sourcebatch, opt) + if jobgraph.add_orders then + _add_jobgraph(target, jobgraph, sourcebatch, opt) else - _add_batchjobs(target, jobs, sourcebatch, opt) + _add_batchjobs(target, jobgraph, sourcebatch, opt) end end diff --git a/xmake/rules/asm/xmake.lua b/xmake/rules/asm/xmake.lua index 78b693137..1c6743793 100644 --- a/xmake/rules/asm/xmake.lua +++ b/xmake/rules/asm/xmake.lua @@ -21,7 +21,7 @@ -- define rule: asm.build rule("asm.build") set_sourcekinds("as") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: asm rule("asm") diff --git a/xmake/rules/cuda/xmake.lua b/xmake/rules/cuda/xmake.lua index 334f569f1..4ce6699ff 100644 --- a/xmake/rules/cuda/xmake.lua +++ b/xmake/rules/cuda/xmake.lua @@ -22,7 +22,7 @@ rule("cuda.build") set_sourcekinds("cu") add_deps("cuda.build.devlink") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- https://github.com/xmake-io/xmake/issues/4755 local cu_ccbin = target:tool("cu-ccbin") diff --git a/xmake/rules/dlang/xmake.lua b/xmake/rules/dlang/xmake.lua index 7ffb8c42f..f5e741f7a 100644 --- a/xmake/rules/dlang/xmake.lua +++ b/xmake/rules/dlang/xmake.lua @@ -21,7 +21,7 @@ rule("dlang.build") set_sourcekinds("dc") add_deps("dlang.build.optimization") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_load(function (target) local toolchains = target:get("toolchains") or get_config("toolchain") if not toolchains or not table.contains(table.wrap(toolchains), "dlang", "dmd", "ldc", "gdc") then diff --git a/xmake/rules/objc++/xmake.lua b/xmake/rules/objc++/xmake.lua index c653261e7..3714ddf3f 100644 --- a/xmake/rules/objc++/xmake.lua +++ b/xmake/rules/objc++/xmake.lua @@ -31,7 +31,7 @@ rule("objc.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc++.build rule("objc++.build") @@ -46,7 +46,7 @@ rule("objc++.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc rule("objc++") diff --git a/xmake/rules/swift/xmake.lua b/xmake/rules/swift/xmake.lua index d1ea0e886..9a2dfbb5b 100644 --- a/xmake/rules/swift/xmake.lua +++ b/xmake/rules/swift/xmake.lua @@ -21,7 +21,7 @@ -- define rule: swift.build rule("swift.build") set_sourcekinds("sc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- we use swift-frontend to support multiple modules -- @see https://github.com/xmake-io/xmake/issues/3916 diff --git a/xmake/rules/winsdk/xmake.lua b/xmake/rules/winsdk/xmake.lua index 0544abb86..6461c4eca 100644 --- a/xmake/rules/winsdk/xmake.lua +++ b/xmake/rules/winsdk/xmake.lua @@ -21,7 +21,7 @@ -- define rule: win.sdk.resource rule("win.sdk.resource") set_sourcekinds("mrc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: application rule("win.sdk.application") diff --git a/xmake/rules/zig/xmake.lua b/xmake/rules/zig/xmake.lua index 4565fcda6..3d0a0a4b9 100644 --- a/xmake/rules/zig/xmake.lua +++ b/xmake/rules/zig/xmake.lua @@ -26,7 +26,7 @@ rule("zig.build") os.mkdir(cachedir) target:add("zcflags", "--cache-dir " .. cachedir) end) - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: zig rule("zig") -- cgit v1.3.1 From c167aa90ca9a0000b7933a9df9306d18dee60f5c Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:41:41 +0800 Subject: build binary --- xmake/actions/build/builtin/build_binary.lua | 127 ++++++++++++++++++++- xmake/actions/build/builtin/linkdepfiles.lua | 39 +++++++ xmake/actions/build/deprecated/kinds/binary.lua | 2 +- .../build/deprecated/kinds/linkdepfiles.lua | 39 ------- xmake/actions/build/deprecated/kinds/shared.lua | 2 +- xmake/actions/build/deprecated/kinds/static.lua | 2 +- 6 files changed, 167 insertions(+), 44 deletions(-) create mode 100644 xmake/actions/build/builtin/linkdepfiles.lua delete mode 100644 xmake/actions/build/deprecated/kinds/linkdepfiles.lua diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua index 89c24a1e7..7b9b7d7b9 100644 --- a/xmake/actions/build/builtin/build_binary.lua +++ b/xmake/actions/build/builtin/build_binary.lua @@ -20,9 +20,132 @@ -- imports import("core.base.option") +import("core.theme.theme") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("private.utils.batchcmds") +import("linkdepfiles", {alias = "get_linkdepfiles"}) import("build_object") +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +-- on link the given target +function _on_link_target(target, opt) + + -- link target with rules + local done = false + for _, r in ipairs(target:orderules()) do + local on_link = r:script("link") + if on_link then + on_link(target, opt) + done = true + end + local on_linkcmd = r:script("linkcmd") + if on_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + on_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + done = true + end + end + if done then return end + + -- do link + _do_link_target(target, opt) +end + +-- link target +function _link_target(target, opt) + + -- do before link for target + local before_link = target:script("link_before") + if before_link then + before_link(target, opt) + end + + -- do before link for rules + for _, r in ipairs(target:orderules()) do + local before_link = r:script("link_before") + if before_link then + before_link(target, opt) + end + local before_linkcmd = r:script("linkcmd_before") + if before_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + + -- on link + target:script("link", _on_link_target)(target, opt) + + -- do after link for target + local after_link = target:script("link_after") + if after_link then + after_link(target, opt) + end + + -- do after link for rules + for _, r in ipairs(target:orderules()) do + local after_link = r:script("link_after") + if after_link then + after_link(target, opt) + end + local after_linkcmd = r:script("linkcmd_after") + if after_linkcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_linkcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end +end + + function main(jobgraph, target) - build_object(jobgraph, target) - -- TODO add link jobs + local objects_group = target:fullname() .. "/objects" + local jobsize = jobgraph:size() + jobgraph:group(objects_group, function () + build_object(jobgraph, target) + end) + if jobgraph:size() > jobsize then + local linkjob = target:fullname() .. "/link" + jobgraph:add(linkjob, function (index, total, opt) + _link_target(target, opt) + end) + jobgraph:add_orders(objects_group, linkjob) + end end diff --git a/xmake/actions/build/builtin/linkdepfiles.lua b/xmake/actions/build/builtin/linkdepfiles.lua new file mode 100644 index 000000000..f96e532fc --- /dev/null +++ b/xmake/actions/build/builtin/linkdepfiles.lua @@ -0,0 +1,39 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file linkdepfiles.lua +-- + +-- get link depfiles +function main(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end diff --git a/xmake/actions/build/deprecated/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua index 3f67232f7..fddc4c8bf 100644 --- a/xmake/actions/build/deprecated/kinds/binary.lua +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) -- do link target function _do_link_target(target, opt) diff --git a/xmake/actions/build/deprecated/kinds/linkdepfiles.lua b/xmake/actions/build/deprecated/kinds/linkdepfiles.lua deleted file mode 100644 index f96e532fc..000000000 --- a/xmake/actions/build/deprecated/kinds/linkdepfiles.lua +++ /dev/null @@ -1,39 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file linkdepfiles.lua --- - --- get link depfiles -function main(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles -end diff --git a/xmake/actions/build/deprecated/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua index cc7aa010c..2b53e98e8 100644 --- a/xmake/actions/build/deprecated/kinds/shared.lua +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) -- do link target function _do_link_target(target, opt) diff --git a/xmake/actions/build/deprecated/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua index abb6c5f72..21fc253b5 100644 --- a/xmake/actions/build/deprecated/kinds/static.lua +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) -- do link target function _do_link_target(target, opt) -- cgit v1.3.1 From 4cebb90c10ea682b66a0f112a0aadddc8b9d8bc2 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:46:03 +0800 Subject: move linkdepfiles --- xmake/actions/build/builtin/build_binary.lua | 4 +-- xmake/actions/build/builtin/linkdepfiles.lua | 39 ------------------------- xmake/actions/build/deprecated/kinds/binary.lua | 4 +-- xmake/actions/build/deprecated/kinds/shared.lua | 4 +-- xmake/actions/build/deprecated/kinds/static.lua | 4 +-- xmake/actions/build/target_utils.lua | 20 +++++++++++++ 6 files changed, 28 insertions(+), 47 deletions(-) delete mode 100644 xmake/actions/build/builtin/linkdepfiles.lua diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua index 7b9b7d7b9..1e51d29f4 100644 --- a/xmake/actions/build/builtin/build_binary.lua +++ b/xmake/actions/build/builtin/build_binary.lua @@ -26,8 +26,8 @@ import("core.tool.compiler") import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") -import("linkdepfiles", {alias = "get_linkdepfiles"}) import("build_object") +import(".target_utils") -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_utils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/builtin/linkdepfiles.lua b/xmake/actions/build/builtin/linkdepfiles.lua deleted file mode 100644 index f96e532fc..000000000 --- a/xmake/actions/build/builtin/linkdepfiles.lua +++ /dev/null @@ -1,39 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file linkdepfiles.lua --- - --- get link depfiles -function main(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles -end diff --git a/xmake/actions/build/deprecated/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua index fddc4c8bf..73962becc 100644 --- a/xmake/actions/build/deprecated/kinds/binary.lua +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) +import("..target_utils") -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_utils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/deprecated/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua index 2b53e98e8..9b91d67e3 100644 --- a/xmake/actions/build/deprecated/kinds/shared.lua +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) +import("..target_utils") -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_utils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/deprecated/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua index 21fc253b5..7057395f9 100644 --- a/xmake/actions/build/deprecated/kinds/static.lua +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..builtin.linkdepfiles", {alias = "get_linkdepfiles"}) +import("..target_utils") -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_utils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index dfb21b23f..1ff2ccdad 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -548,6 +548,26 @@ function get_filejobs(targets_root, opt) return jobgraph end +-- get link depfiles +function get_linkdepfiles(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end + -- get all root targets function get_root_targets(targetnames, opt) opt = opt or {} -- cgit v1.3.1 From b0c4c943920b758b7f9482952e5227f224301af4 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 23:14:19 +0800 Subject: add link objects --- xmake/actions/build/builtin/build_binary.lua | 121 +---------------------- xmake/actions/build/builtin/build_moduleonly.lua | 3 +- xmake/actions/build/builtin/build_object.lua | 1 - xmake/actions/build/builtin/build_shared.lua | 3 +- xmake/actions/build/builtin/build_static.lua | 3 +- xmake/actions/build/builtin/link_objects.lua | 68 +++++++++++++ xmake/actions/build/target_utils.lua | 12 +++ 7 files changed, 90 insertions(+), 121 deletions(-) create mode 100644 xmake/actions/build/builtin/link_objects.lua diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua index 1e51d29f4..3eb376757 100644 --- a/xmake/actions/build/builtin/build_binary.lua +++ b/xmake/actions/build/builtin/build_binary.lua @@ -19,122 +19,9 @@ -- -- imports -import("core.base.option") -import("core.theme.theme") -import("core.tool.linker") -import("core.tool.compiler") -import("core.project.depend") -import("utils.progress") -import("private.utils.batchcmds") import("build_object") import(".target_utils") --- do link target -function _do_link_target(target, opt) - local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) - local linkflags = linkinst:linkflags({target = target}) - - -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) - local dryrun = option.get("dry-run") - local depvalues = {linkinst:program(), linkflags} - depend.on_changed(function () - local filename = target:filename() - if target:namespace() then - filename = target:namespace() .. "::" .. filename - end - progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) - - local targetfile = target:targetfile() - local objectfiles = target:objectfiles() - local verbose = option.get("verbose") - if verbose then - -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows - print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) - end - - if not dryrun then - assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) - end - end, {dependfile = target:dependfile(), - lastmtime = os.mtime(target:targetfile()), - changed = target:is_rebuilt() or option.get("linkonly"), - values = depvalues, files = depfiles, dryrun = dryrun}) -end - --- on link the given target -function _on_link_target(target, opt) - - -- link target with rules - local done = false - for _, r in ipairs(target:orderules()) do - local on_link = r:script("link") - if on_link then - on_link(target, opt) - done = true - end - local on_linkcmd = r:script("linkcmd") - if on_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - on_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - done = true - end - end - if done then return end - - -- do link - _do_link_target(target, opt) -end - --- link target -function _link_target(target, opt) - - -- do before link for target - local before_link = target:script("link_before") - if before_link then - before_link(target, opt) - end - - -- do before link for rules - for _, r in ipairs(target:orderules()) do - local before_link = r:script("link_before") - if before_link then - before_link(target, opt) - end - local before_linkcmd = r:script("linkcmd_before") - if before_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - - -- on link - target:script("link", _on_link_target)(target, opt) - - -- do after link for target - local after_link = target:script("link_after") - if after_link then - after_link(target, opt) - end - - -- do after link for rules - for _, r in ipairs(target:orderules()) do - local after_link = r:script("link_after") - if after_link then - after_link(target, opt) - end - local after_linkcmd = r:script("linkcmd_after") - if after_linkcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_linkcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end -end - - function main(jobgraph, target) local objects_group = target:fullname() .. "/objects" local jobsize = jobgraph:size() @@ -142,10 +29,10 @@ function main(jobgraph, target) build_object(jobgraph, target) end) if jobgraph:size() > jobsize then - local linkjob = target:fullname() .. "/link" - jobgraph:add(linkjob, function (index, total, opt) - _link_target(target, opt) + local link_group = target:fullname() .. "/link" + jobgraph:group(link_group, function () + target_utils.add_linkjobs(jobgraph, target) end) - jobgraph:add_orders(objects_group, linkjob) + jobgraph:add_orders(objects_group, link_group) end end diff --git a/xmake/actions/build/builtin/build_moduleonly.lua b/xmake/actions/build/builtin/build_moduleonly.lua index b45766f4c..e3cc87c27 100644 --- a/xmake/actions/build/builtin/build_moduleonly.lua +++ b/xmake/actions/build/builtin/build_moduleonly.lua @@ -19,7 +19,8 @@ -- -- imports -import("core.base.option") +import("build_object") function main(jobgraph, target) + build_object(jobgraph, target) end diff --git a/xmake/actions/build/builtin/build_object.lua b/xmake/actions/build/builtin/build_object.lua index f12dce686..82268c1e7 100644 --- a/xmake/actions/build/builtin/build_object.lua +++ b/xmake/actions/build/builtin/build_object.lua @@ -19,7 +19,6 @@ -- -- imports -import("core.base.option") import(".target_utils") function main(jobgraph, target) diff --git a/xmake/actions/build/builtin/build_shared.lua b/xmake/actions/build/builtin/build_shared.lua index a178dafd6..077489710 100644 --- a/xmake/actions/build/builtin/build_shared.lua +++ b/xmake/actions/build/builtin/build_shared.lua @@ -19,7 +19,8 @@ -- -- imports -import("core.base.option") +import("build_binary") function main(jobgraph, target) + build_binary(jobgraph, target) end diff --git a/xmake/actions/build/builtin/build_static.lua b/xmake/actions/build/builtin/build_static.lua index 21de325da..8b9b2ffaf 100644 --- a/xmake/actions/build/builtin/build_static.lua +++ b/xmake/actions/build/builtin/build_static.lua @@ -19,7 +19,8 @@ -- -- imports -import("core.base.option") +import("build_binary") function main(jobgraph, target) + build_binary(jobgraph, target) end diff --git a/xmake/actions/build/builtin/link_objects.lua b/xmake/actions/build/builtin/link_objects.lua new file mode 100644 index 000000000..13a38d0cf --- /dev/null +++ b/xmake/actions/build/builtin/link_objects.lua @@ -0,0 +1,68 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file link_objects.lua +-- + +-- imports +import("core.base.option") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("build_object") +import(".target_utils") + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = target_utils.get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +function main(jobgraph, target) + local linkjob = target:fullname() .. "/link_objects" + jobgraph:add(linkjob, function (index, total, opt) + _do_link_target(target, opt) + end) +end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 1ff2ccdad..47c4aa9ec 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -104,6 +104,8 @@ function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then if job_kind == "prepare" then import("builtin.prepare_files", {anonymous = true})(jobgraph, target) + elseif job_kind == "link" then + import("builtin.link_objects", {anonymous = true})(jobgraph, target) else import("builtin.build_" .. target:kind(), {anonymous = true})(jobgraph, target) end @@ -548,6 +550,16 @@ function get_filejobs(targets_root, opt) return jobgraph end +-- add link jobs for the given target +function add_linkjobs(jobgraph, target, opt) + opt = opt or {} + opt.job_kind = "link" + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + jobgraph:add_orders(group_before, group, group_after) +end + -- get link depfiles function get_linkdepfiles(target) local extrafiles = {} -- cgit v1.3.1 From 2e6cc42d5cc78ce8df9e50d3cffbdd446a26ec8a Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 23:27:44 +0800 Subject: fix add_filejobs --- xmake/actions/build/target_utils.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 47c4aa9ec..1d2318a5e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -494,8 +494,13 @@ function add_filejobs(jobgraph, target, opt) end -- get sourcebatches + local sourcebatches local filepatterns = opt.filepatterns - local sourcebatches = filepatterns and _match_sourcebatches(target, filepatterns) or target:sourcebatches() + if filepatterns then + sourcebatches = _match_sourcebatches(target, filepatterns) + else + sourcebatches = target:sourcebatches() + end -- we just build sourcebatch with on_build_files scripts -- -- cgit v1.3.1 From b644fa46c237f5471e313dfa28f4ffc11cec956f Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 23:45:03 +0800 Subject: enable jobgraph for verilator --- xmake/rules/verilator/verilator.lua | 4 ++-- xmake/rules/verilator/xmake.lua | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/xmake/rules/verilator/verilator.lua b/xmake/rules/verilator/verilator.lua index 6b7267861..0a87c40ff 100644 --- a/xmake/rules/verilator/verilator.lua +++ b/xmake/rules/verilator/verilator.lua @@ -207,7 +207,7 @@ endmodule]]) os.rm(tmpdir) end -function build_cppfiles(target, batchjobs, sourcebatch, opt) +function build_cppfiles(target, jobgraph, sourcebatch, opt) local toolchain = assert(target:toolchain("verilator"), 'we need to set_toolchains("verilator") in target("%s")', target:name()) local verilator = assert(toolchain:config("verilator"), "verilator not found!") local autogendir = path.join(target:autogendir(), "rules", "verilator") @@ -261,7 +261,7 @@ function build_cppfiles(target, batchjobs, sourcebatch, opt) table.insert(sourcebatch_cpp.objectfiles, objectfile) table.insert(sourcebatch_cpp.dependfiles, dependfile) end - build_objectfiles(target, batchjobs, sourcebatch_cpp, opt) + build_objectfiles(target, jobgraph, sourcebatch_cpp, opt) end function buildcmd_vfiles(target, batchcmds, sourcebatch, opt) diff --git a/xmake/rules/verilator/xmake.lua b/xmake/rules/verilator/xmake.lua index 4e6f49397..ca60a3289 100644 --- a/xmake/rules/verilator/xmake.lua +++ b/xmake/rules/verilator/xmake.lua @@ -34,9 +34,9 @@ rule("verilator.binary") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) @@ -61,9 +61,9 @@ rule("verilator.static") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) -- cgit v1.3.1 From 736e327bf3b51888da68cba78f4ee1b780a7b1d6 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 30 Mar 2025 01:05:28 +0800 Subject: improve protobuf --- xmake/core/base/graph.lua | 5 +++ xmake/rules/protobuf/proto.lua | 93 +----------------------------------------- xmake/rules/protobuf/xmake.lua | 14 ++----- 3 files changed, 11 insertions(+), 101 deletions(-) diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index db8682b21..53f09c59f 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -318,6 +318,11 @@ function graph:topo_sort() return order_vertices, has_cycle end +-- deprecated +function graph:topological_sort() + return self:topo_sort() +end + -- find cycle function graph:find_cycle() local visited = {} diff --git a/xmake/rules/protobuf/proto.lua b/xmake/rules/protobuf/proto.lua index c35ffb697..bebeb3105 100644 --- a/xmake/rules/protobuf/proto.lua +++ b/xmake/rules/protobuf/proto.lua @@ -22,10 +22,8 @@ import("core.base.option") import("lib.detect.find_tool") import("core.project.depend") -import("private.action.build.object", {alias = "build_objectfiles"}) import("utils.progress") import("private.utils.batchcmds") -import("private.async.buildjobs") -- get protoc function _get_protoc(target, sourcekind) @@ -100,7 +98,7 @@ function load(target, sourcekind) end end -function buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) +function buildcmd_pfile(target, batchcmds, sourcefile_proto, sourcekind, opt) -- get protoc local protoc = _get_protoc(target, sourcekind) @@ -166,7 +164,7 @@ function buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) end end -function buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) +function buildcmd_cxfile(target, batchcmds, sourcefile_proto, sourcekind, opt) -- get protoc local protoc = _get_protoc(target, sourcekind) @@ -223,90 +221,3 @@ function buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) end end -function build_cxfile_objects(target, batchjobs, opt, sourcekind) - local sourcebatch_cx = { - rulename = (sourcekind == "cxx" and "c++" or "c").. ".build", - sourcekind = sourcekind, - sourcefiles = {}, - objectfiles = {}, - dependfiles = {} - } - for _, sourcefile_proto in ipairs(sourcefiles) do - -- get c/c++ source file for protobuf - local prefixdir - local autogendir - local public - local grpc_cpp_plugin - local fileconfig = target:fileconfig(sourcefile_proto) - if fileconfig then - public = fileconfig.proto_public - prefixdir = fileconfig.proto_rootdir - -- custom autogen directory to access the generated header files - -- @see https://github.com/xmake-io/xmake/issues/3678 - autogendir = fileconfig.proto_autogendir - grpc_cpp_plugin = fileconfig.proto_grpc_cpp_plugin - end - local rootdir = autogendir and autogendir or path.join(target:autogendir(), "rules", "protobuf") - local filename = path.basename(sourcefile_proto) .. ".pb" .. (sourcekind == "cxx" and ".cc" or "-c.c") - local sourcefile_cx = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename}) - local sourcefile_dir = prefixdir and path.join(rootdir, prefixdir) or path.directory(sourcefile_cx) - - local grpc_cpp_plugin_bin - local filename_grpc - local sourcefile_cx_grpc - if grpc_cpp_plugin then - grpc_cpp_plugin_bin = _get_grpc_cpp_plugin(target, sourcekind) - filename_grpc = path.basename(sourcefile_proto) .. ".grpc.pb.cc" - sourcefile_cx_grpc = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename_grpc}) - end - - -- add includedirs - target:add("includedirs", sourcefile_dir, {public = public}) - - -- add objectfile - local objectfile = target:objectfile(sourcefile_cx) - local dependfile = target:dependfile(sourcefile_proto) - table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx) - table.insert(sourcebatch_cx.objectfiles, objectfile) - table.insert(sourcebatch_cx.dependfiles, dependfile) - - local objectfile_grpc - if grpc_cpp_plugin then - objectfile_grpc = target:objectfile(sourcefile_cx_grpc) - table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx_grpc) - table.insert(sourcebatch_cx.objectfiles, objectfile_grpc) - table.insert(sourcebatch_cx.dependfiles, dependfile) - end - end - build_objectfiles(target, batchjobs, sourcebatch_cx, opt) -end - --- build batch jobs -function build_cxfiles(target, batchjobs, sourcebatch, opt, sourcekind) - opt = opt or {} - local nodes = {} - local nodenames = {} - local node_rulename = "rules/" .. sourcebatch.rulename .. "/node" - local sourcefiles = sourcebatch.sourcefiles - for _, sourcefile_proto in ipairs(sourcefiles) do - local nodename = node_rulename .. "/" .. sourcefile_proto - nodes[nodename] = { - name = nodename, - job = batchjobs:addjob(nodename, function(index, total, jobopt) - local batchcmds_ = batchcmds.new({target = target}) - buildcmd_pfiles(target, batchcmds_, sourcefile_proto, {progress = jobopt.progress}, sourcekind) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - } - table.insert(nodenames, nodename) - end - local rootname = "rules/" .. sourcebatch.rulename .. "/root" - nodes[rootname] = { - name = rootname, - deps = nodenames, - job = batchjobs:addjob(rootname, function(_index, _total) - build_cxfile_objects(target, batchjobs, opt, sourcekind) - end) - } - buildjobs(nodes, batchjobs, opt.rootjob) -end diff --git a/xmake/rules/protobuf/xmake.lua b/xmake/rules/protobuf/xmake.lua index 119ff3d1f..f2cbf2615 100644 --- a/xmake/rules/protobuf/xmake.lua +++ b/xmake/rules/protobuf/xmake.lua @@ -27,14 +27,11 @@ rule("protobuf.cpp") end) -- generate build commands before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cxx") + import("proto").buildcmd_pfile(target, batchcmds, sourcefile_proto, "cxx", opt) end) on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cxx") + import("proto").buildcmd_cxfile(target, batchcmds, sourcefile_proto, "cxx", opt) end) - before_build_files(function (target, batchjobs, sourcebatch, opt) - import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cxx") - end, {batch = true}) -- define rule: protobuf.c @@ -45,11 +42,8 @@ rule("protobuf.c") import("proto").load(target, "cc") end) before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cc") + import("proto").buildcmd_pfile(target, batchcmds, sourcefile_proto, "cc", opt) end) on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cc") + import("proto").buildcmd_cxfile(target, batchcmds, sourcefile_proto, "cc", opt) end) - before_build_files(function (target, batchjobs, sourcebatch, opt) - import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cc") - end, {batch = true}) -- cgit v1.3.1 From bfb940fb7f5c4e89d2e0d2ececbd2c557ee04f7e Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 30 Mar 2025 22:52:32 +0800 Subject: fix cmd scripts --- xmake/actions/build/target_utils.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 1d2318a5e..87412e1b8 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -432,6 +432,8 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" + local job_kindcmd_file = job_kind .. "cmd_file" + local job_kindcmd_files = job_kind .. "cmd_files" -- the group name, e.g. foo/after_prepare_files, bar/before_build_files local group_name = string.format("%s/%s_%s_files", target:fullname(), stage, job_kind) @@ -441,8 +443,8 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files - local scriptcmd_file_name = stage ~= "" and (job_kind_file .. "cmd_" .. stage) or (job_kind_file .. "cmd") - local scriptcmd_files_name = stage ~= "" and (job_kind_files .. "cmd_" .. stage) or (job_kind_files .. "cmd") + local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file + local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files -- build sourcebatches map local sourcebatches_map = {} @@ -514,6 +516,7 @@ function add_filejobs(jobgraph, target, opt) local rulename = sourcebatch.rulename if rulename then local ruleinst = _get_rule(target, rulename) + -- FIXME if ruleinst:script("build_file") or ruleinst:script("build_files") then table.insert(sourcebatches_result, sourcebatch) end -- cgit v1.3.1 From 147ef59b5d5aeb19519e98ed07179ff84edaa5c9 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 22:32:02 +0800 Subject: fix rule batches --- xmake/actions/build/target_utils.lua | 56 ++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 87412e1b8..6eddda91f 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -448,10 +448,26 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) -- build sourcebatches map local sourcebatches_map = {} - for _, sourcebatch in ipairs(sourcebatches) do - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = _get_rule(target, rulename) - sourcebatches_map[ruleinst] = sourcebatch + local sourcebatches_for_target = {} + for _, sourcebatch in pairs(sourcebatches) do + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = _get_rule(target, rulename) + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + else + table.insert(sourcebatches_for_target, sourcebatch) + end end -- TODO sort rules and jobs @@ -471,7 +487,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) } for _, instance in ipairs(instances) do if instance == target then - for _, sourcebatch in ipairs(sourcebatches) do + for _, sourcebatch in ipairs(sourcebatches_for_target) do add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) end else -- rule @@ -504,34 +520,10 @@ function add_filejobs(jobgraph, target, opt) sourcebatches = target:sourcebatches() end - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - local sourcebatches_result = {} - for _, sourcebatch in pairs(sourcebatches) do - local rulename = sourcebatch.rulename - if rulename then - local ruleinst = _get_rule(target, rulename) - -- FIXME - if ruleinst:script("build_file") or ruleinst:script("build_files") then - table.insert(sourcebatches_result, sourcebatch) - end - else - table.insert(sourcebatches_result, sourcebatch) - end - end - if #sourcebatches_result == 0 then - return - end - -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "", opt) - local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "before", opt) - local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches_result, "after", opt) + local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) jobgraph:add_orders(group_before, group, group_after) end -- cgit v1.3.1 From 1b735b328bda4d207675a1406dfe714e934de0c8 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 22:50:16 +0800 Subject: improve c++modules for jobgraph --- xmake/actions/build/target_utils.lua | 4 +- .../rules/c++/modules/modules_support/builder.lua | 83 ++++++++++++++-- .../c++/modules/modules_support/clang/builder.lua | 104 +++++++++++++++++++- .../modules/modules_support/dependency_scanner.lua | 2 +- .../c++/modules/modules_support/gcc/builder.lua | 107 ++++++++++++++++++++- .../c++/modules/modules_support/msvc/builder.lua | 101 ++++++++++++++++++- xmake/rules/c++/modules/xmake.lua | 24 +++-- 7 files changed, 398 insertions(+), 27 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 6eddda91f..798009ab6 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -292,7 +292,7 @@ end -- get target jobs function get_targetjobs(targets_root, opt) - local jobgraph = async_jobgraph.new() + local jobgraph = async_jobgraph.new(opt.job_kind) local targetrefs = {} for _, target in ipairs(targets_root) do add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) @@ -542,7 +542,7 @@ end -- get files jobs function get_filejobs(targets_root, opt) - local jobgraph = async_jobgraph.new() + local jobgraph = async_jobgraph.new(opt.job_kind) local targetrefs = {} for _, target in ipairs(targets_root) do add_filejobs_and_deps(jobgraph, target, targetrefs, opt) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a71e2c392..0139bc107 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -34,6 +34,7 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles + local jobgraph = opt.jobgraph for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then @@ -45,7 +46,8 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + table.insert(deps, depname) end opt.build_module(deps, module, name, objectfile, cppfile) @@ -204,7 +206,6 @@ end -- "file": "foo.cppm" -- } function _generate_meta_module_info(target, name, sourcefile, requires) - local modulehash = compiler_support.get_modulehash(target, sourcefile) local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} @@ -266,7 +267,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "_populate_module_map" + local populate_jobname = target:name() .. "/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -278,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = name and target:name() .. name or cppfile + local job_name = target:fullname() .. "/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -288,9 +289,39 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) end +-- build modules for jobgraph +function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local jobdeps = {} + local build_modules_group = target:fullname() .. "/build_modules" + jobgraph:group(build_modules_group, function () + + -- add populate module job + local populate_jobname = target:fullname() .. "/populate_module_map" + jobgraph:add(populate_jobname, function(index, total, opt) + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + end) + + -- add module jobs + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, module, name, objectfile, cppfile) + local jobname = target:fullname() .. "/" .. (name or cppfile) + _builder(target).make_module_jobgraph(target, jobgraph, { + module = module, objectfile = objectfile, cppfile = cppfile + }) + jobdeps[jobname] = table.join(populate_jobname, deps) + end}) + ) + end) + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end +end + -- build modules for batchcmds function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local depmtime = 0 opt.progress = opt.progress or 0 @@ -307,7 +338,7 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op batchcmds:set_depmtime(depmtime) end --- generate headerunits for batchjobs +-- build headerunits for batchjobs function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -345,7 +376,43 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules end end --- generate headerunits for batchcmds +-- build headerunits for jobgraph +function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + local build_modules_group = target:fullname() .. "/build_modules" + local build_headerunits_group = target:fullname() .. "/build_headerunits" + jobgraph:group(build_headerunits_group, function () + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:fullname() .. "/" .. key + _builder(target).make_headerunit_buildjobs(target, + job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + end + })) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end + end) + jobgraph:add_orders(build_headerunits_group, build_modules_group) +end + +-- build headerunits for batchcmds function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -391,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 972694254..fa2d46844 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:name() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -278,6 +278,75 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local is_mapped_bmi = mapped_bmi ~= nil + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -351,6 +420,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show(jobopt.progress, + "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", + target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 94c14e60d..e576bd562 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -220,7 +220,7 @@ end -- generate dependency files function _generate_dependencies(target, sourcebatch, opt) local changed = false - if opt.batchjobs then + if opt.jobgraph then local jobs = option.get("jobs") or os.default_njob() runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index a0a43db5a..20cda94f8 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -241,6 +241,69 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + end + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + if sourcefile then + _compile(target, flags, sourcefile, opt.objectfile) + end + os.tryrm(module_mapper) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -337,6 +400,46 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + if option.get("diagnosis") then + print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _compile(target, + _make_headerunitflags(target, headerunit, headerunit_mapper, opt), + path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + + + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 5998db62f..cb2dcad1d 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -257,16 +257,15 @@ end -- build module file for batchjobs function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then @@ -326,6 +325,71 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -401,6 +465,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f967dd133..30d65cf2a 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -71,7 +71,7 @@ rule("c++.build.modules.builder") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") -- parallel build support to accelerate `xmake build` to build modules - before_build_files(function(target, batchjobs, sourcebatch, opt) + before_build_files(function(target, jobgraph, sourcebatch, opt) if target:data("cxx.has_modules") then import("modules_support.compiler_support") import("modules_support.dependency_scanner") @@ -102,7 +102,7 @@ rule("c++.build.modules.builder") end opt = opt or {} - opt.batchjobs = true + opt.jobgraph = true compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) @@ -112,11 +112,17 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build modules - builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - - -- build headerunits and we need to do it before building modules - builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + if jobgraph.add_orders then + -- build modules + builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + else + -- build modules, deprecated + builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + end sourcebatch.objectfiles = link_objectfiles else @@ -129,7 +135,7 @@ rule("c++.build.modules.builder") -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end - end, {batch = true}) + end, {jobgraph = true, batch = true}) -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) @@ -163,7 +169,7 @@ rule("c++.build.modules.builder") end opt = opt or {} - opt.batchjobs = false + opt.jobgraph = false compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) -- cgit v1.3.1 From 8b0ba84af0611cc61e9398c579ab55e9087320d4 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 22:51:19 +0800 Subject: use target:fullname() --- .../rules/c++/modules/modules_support/builder.lua | 34 +++++++++++----------- .../c++/modules/modules_support/clang/builder.lua | 28 +++++++++--------- .../modules_support/clang/dependency_scanner.lua | 2 +- .../modules/modules_support/compiler_support.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 8 ++--- .../c++/modules/modules_support/gcc/builder.lua | 28 +++++++++--------- .../modules_support/gcc/dependency_scanner.lua | 2 +- .../c++/modules/modules_support/msvc/builder.lua | 28 +++++++++--------- .../modules_support/msvc/dependency_scanner.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 6 ++-- 10 files changed, 70 insertions(+), 70 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 0139bc107..59f00b64f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -130,7 +130,7 @@ function _try_reuse_modules(target, modules) end local mapped = get_from_target_mapper(dep, name) if mapped then - compiler_support.memcache():set2(target:name() .. name, "reuse", true) + compiler_support.memcache():set2(target:fullname() .. name, "reuse", true) add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) break end @@ -158,8 +158,8 @@ function should_build(target, sourcefile, bmifile, opt) for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) if m then - local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) - or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key) + local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) + or compiler_support.memcache():get2("should_build_in_" .. target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -174,7 +174,7 @@ function should_build(target, sourcefile, bmifile, opt) if opt.name then local m = get_from_target_mapper(target, opt.name) if m and m.opt and m.opt.target then - local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -224,7 +224,7 @@ end function _target_module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end function _is_duplicated_headerunit(target, key) @@ -252,7 +252,7 @@ function _builder(target) end function mark_build(target, name) - compiler_support.memcache():set2("should_build_in_" .. target:name(), name, true) + compiler_support.memcache():set2("should_build_in_" .. target:fullname(), name, true) end -- build batchjobs for modules @@ -263,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -349,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:name() .. key + local job_name = target:fullname() .. "/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -462,7 +462,7 @@ function generate_metadata(target, modules) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) - progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name) local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) json.savefile(metafilepath, metadata) end, {comax = jobs, total = #public_modules}) @@ -471,20 +471,20 @@ end -- flush target module mapper keys function flush_target_module_mapper_keys(target) local memcache = compiler_support.memcache() - memcache:set2(target:name(), "module_mapper_keys", nil) + memcache:set2(target:fullname(), "module_mapper_keys", nil) end -- get or create a target module mapper function get_target_module_mapper(target) local memcache = compiler_support.memcache() - local mapper = memcache:get2(target:name(), "module_mapper") + local mapper = memcache:get2(target:fullname(), "module_mapper") if not mapper then mapper = {} - memcache:set2(target:name(), "module_mapper", mapper) + memcache:set2(target:fullname(), "module_mapper", mapper) end -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit - local mapper_keys = memcache:get2(target:name(), "module_mapper_keys") + local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys") if not mapper_keys then mapper_keys = {} for _, item in pairs(mapper) do @@ -492,7 +492,7 @@ function get_target_module_mapper(target) mapper_keys[item.key] = item end end - memcache:set2(target:name(), "module_mapper_keys", mapper_keys) + memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys) end return mapper, mapper_keys end @@ -530,7 +530,7 @@ end -- check if dependencies changed function is_dependencies_changed(target, module) - local cachekey = target:name() .. module.name + local cachekey = target:fullname() .. module.name local requires = hashset.from(table.keys(module.requires or {})) local oldrequires = compiler_support.memcache():get2(cachekey, "oldrequires") local changed = false diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index fa2d46844..dd9a6fad5 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -134,7 +134,7 @@ function _get_requiresflags(target, module, opt) local modulefileflag = compiler_support.get_modulefileflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -216,11 +216,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -263,11 +263,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) end else @@ -288,7 +288,7 @@ function make_module_jobgraph(target, jobgraph, opt) local jobname = target:fullname() .. "/" .. (name or opt.cppfile) jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -331,11 +331,11 @@ function make_module_jobgraph(target, jobgraph, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) end else @@ -354,7 +354,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -374,11 +374,11 @@ function make_module_buildcmds(target, batchcmds, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds, is_mapped_bmi = is_mapped_bmi}) end else @@ -409,7 +409,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local depvalues = {compinst:program(), compflags} if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end @@ -440,7 +440,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil if opt.build then progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:name(), headerunit.name) + target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end @@ -458,7 +458,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 06b522ce8..715e8a07d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -36,7 +36,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index cf64a08bb..a5a32e2c5 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -224,7 +224,7 @@ function modules_cachedir(target, opt) end function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:name() + local key = path.directory(modulepath) .. target:fullname() return hash.uuid(key):split("-", {plain = true})[1]:lower() end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index e576bd562..4a8f08909 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -222,7 +222,7 @@ function _generate_dependencies(target, sourcebatch, opt) local changed = false if opt.jobgraph then local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) + runjobs(target:fullname() .. "/module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed end, {comax = jobs, total = #sourcebatch.sourcefiles}) @@ -235,7 +235,7 @@ function _generate_dependencies(target, sourcebatch, opt) end -- get module dependencies function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:name() .. "/" .. sourcebatch.rulename + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename local modules = compiler_support.memcache():get2("modules", cachekey) if modules == nil then modules = compiler_support.localcache():get2("modules", cachekey) @@ -476,8 +476,8 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) objectfiles_sorted_set:remove(objectfile) if name ~= "std" and name ~= "std.compat" then culleds = culleds or {} - culleds[target:name()] = culleds[target:name()] or {} - table.insert(culleds[target:name()], format("%s -> %s", name, cppfile)) + culleds[target:fullname()] = culleds[target:fullname()] or {} + table.insert(culleds[target:fullname()], format("%s -> %s", name, cppfile)) end end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 20cda94f8..38a9e3d04 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -76,7 +76,7 @@ end function _module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end -- generate a module mapper file for build a headerunit @@ -135,7 +135,7 @@ end -- function _generate_modulemapper_file(target, module, cppfile) local maplines = _get_maplines(target, module) - local mapper_path = path.join(os.tmpdir(), target:name():replace(" ", "_"), name or cppfile:replace(" ", "_")) + local mapper_path = path.join(os.tmpdir(), target:fullname():replace(" ", "_"), name or cppfile:replace(" ", "_")) local mapper_content = {} table.insert(mapper_content, "root " .. path.unix(os.projectdir())) for _, mapline in ipairs(maplines) do @@ -186,7 +186,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) sourcefile = opt.cppfile, job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -217,13 +217,13 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -249,7 +249,7 @@ function make_module_jobgraph(target, jobgraph, opt) jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -280,13 +280,13 @@ function make_module_jobgraph(target, jobgraph, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -312,7 +312,7 @@ function make_module_buildcmds(target, batchcmds, opt) local module_mapperflag = compiler_support.get_modulemapperflag(target) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -335,13 +335,13 @@ function make_module_buildcmds(target, batchcmds, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -383,7 +383,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end @@ -421,7 +421,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end @@ -452,7 +452,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) if option.get("diagnosis") then batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) end diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 11dab5669..e087c6d4e 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index cb2dcad1d..fe1f25bb4 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -167,7 +167,7 @@ function _get_requiresflags(target, module, opt) local headerunitflag = compiler_support.get_headerunitflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -175,7 +175,7 @@ function _get_requiresflags(target, module, opt) local deps_flags = {} for required in requires:orderitems() do local dep_module = get_from_target_mapper(target, required) - assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) + assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:fullname()) local mapflag local bmifile = dep_module.bmi @@ -268,7 +268,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -310,11 +310,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else @@ -333,7 +333,7 @@ function make_module_jobgraph(target, jobgraph, opt) jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -375,11 +375,11 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else @@ -397,7 +397,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -417,11 +417,11 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, provide, {batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) end else @@ -454,7 +454,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local name = headerunit.unique and headerunit.name or headerunit.path if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end @@ -485,7 +485,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil local name = headerunit.unique and headerunit.name or headerunit.path if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end @@ -503,7 +503,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), target:objectfile(headerunit.path)) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index 491a27cbf..044510fab 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) local changed = false depend.on_changed(function () - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) local outputdir = compiler_support.get_outputdir(target, sourcefile) local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 30d65cf2a..f4b7f744e 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -129,7 +129,7 @@ rule("c++.build.modules.builder") sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs @@ -191,7 +191,7 @@ rule("c++.build.modules.builder") sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else sourcebatch.sourcefiles = {} @@ -228,7 +228,7 @@ rule("c++.build.modules.install") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets if compiler_support.contains_modules(target) then - local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + local modules = compiler_support.localcache():get2(target:fullname(), "c++.modules") builder.generate_metadata(target, modules) compiler_support.add_installfiles_for_modules(target) -- cgit v1.3.1 From c0867ffd08915f2f80decbf97da17b685fb74525 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 23:05:27 +0800 Subject: fix build modules --- xmake/modules/private/action/build/object.lua | 2 +- .../rules/c++/modules/modules_support/builder.lua | 26 +++++++++++----------- .../c++/modules/modules_support/clang/builder.lua | 6 ++--- .../c++/modules/modules_support/gcc/builder.lua | 7 +++--- .../c++/modules/modules_support/msvc/builder.lua | 7 +++--- xmake/rules/c++/modules/xmake.lua | 11 +++------ 6 files changed, 28 insertions(+), 31 deletions(-) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 5118dbb82..0028b4b6f 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -165,7 +165,7 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) local objectfile = sourcebatch.objectfiles[i] local dependfile = sourcebatch.dependfiles[i] local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) - local jobname = target:fullname() .. "/" .. sourcefile + local jobname = target:fullname() .. "/obj/" .. sourcefile jobgraph:add(jobname, function (index, total, jobopt) local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) build_object(target, sourcefile, build_opt) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 59f00b64f..0fd9920b1 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -46,7 +46,7 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep table.insert(deps, depname) end @@ -263,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -279,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/" .. (name or cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -292,11 +292,11 @@ end -- build modules for jobgraph function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) local jobdeps = {} - local build_modules_group = target:fullname() .. "/build_modules" + local build_modules_group = target:fullname() .. "/module/build_modules" jobgraph:group(build_modules_group, function () -- add populate module job - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" jobgraph:add(populate_jobname, function(index, total, opt) _try_reuse_modules(target, modules) _builder(target).populate_module_map(target, modules) @@ -305,7 +305,7 @@ function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local jobname = target:fullname() .. "/" .. (name or cppfile) + local jobname = target:fullname() .. "/module/" .. (name or cppfile) _builder(target).make_module_jobgraph(target, jobgraph, { module = module, objectfile = objectfile, cppfile = cppfile }) @@ -349,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -385,14 +385,14 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local build_modules_group = target:fullname() .. "/build_modules" - local build_headerunits_group = target:fullname() .. "/build_headerunits" + local build_modules_group = target:fullname() .. "/module/build_modules" + local build_headerunits_group = target:fullname() .. "/module/build_headerunits" jobgraph:group(build_headerunits_group, function () local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key _builder(target).make_headerunit_buildjobs(target, job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) end @@ -458,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index dd9a6fad5..ec6692c35 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -285,7 +285,7 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 38a9e3d04..05e8380da 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -247,7 +247,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index fe1f25bb4..8f582438b 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -263,9 +263,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then @@ -331,7 +331,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f4b7f744e..42eab0c19 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -112,15 +112,12 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles + -- build modules and headerunits, and we need to build headerunits first if jobgraph.add_orders then - -- build modules builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - else - -- build modules, deprecated + else -- deprecated builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) end @@ -179,10 +176,8 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build headerunits + -- build headerunits and modules builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - - -- build modules builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles -- cgit v1.3.1 From 530148989c8d444e78e0b1d8be4157cf5b3e8c46 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 1 Apr 2025 00:41:42 +0800 Subject: fix custom on_xxx in target --- xmake/actions/build/target_utils.lua | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 798009ab6..b242c0a58 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -203,6 +203,10 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then has_script = true end + -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules + if has_script and instance == target and stage == "" then + break + end end -- call builtin script, e.g. on_prepare, on_build, ... @@ -300,7 +304,7 @@ function get_targetjobs(targets_root, opt) return jobgraph end --- add file jobs for the given script, TODO on single file +-- add file jobs for the given script function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) opt = opt or {} local has_script = false @@ -485,12 +489,17 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) scriptcmd_file_name = scriptcmd_file_name, scriptcmd_files_name = scriptcmd_files_name } + local has_target_script = false for _, instance in ipairs(instances) do if instance == target then for _, sourcebatch in ipairs(sourcebatches_for_target) do - add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end end - else -- rule + elseif not has_target_script then -- rule local sourcebatch = sourcebatches_map[instance] if sourcebatch then add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) -- cgit v1.3.1 From f4706b4b526fe88f5849eb0daa3a4d0713862d0c Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 1 Apr 2025 00:51:31 +0800 Subject: fix script name --- xmake/actions/build/target_utils.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index b242c0a58..3a1013ef9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -139,7 +139,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) -- target("test") -- on_build(function (target, opt) -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_name) jobgraph:add(jobname, function (index, total, opt) script(target, {progress = opt.progress}) end) @@ -158,7 +158,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) local scriptcmd_name = opt.scriptcmd_name local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = opt.progress}) @@ -333,7 +333,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) -- target("test") -- on_build_files(function (target, sourcebatch, opt) -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_files_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_files_name) jobgraph:add(jobname, function (index, total, opt) script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) end) @@ -369,7 +369,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) -- target("test") -- on_build_file(function (target, sourcefile, opt) -- end) - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_file_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_file_name) jobgraph:add(jobname, function (index, total, opt) local sourcekind = sourcebatch.sourcekind for _, sourcefile in ipairs(sourcebatch.sourcefiles) do @@ -392,7 +392,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local scriptcmd_files = instance:script(scriptcmd_files_name) if scriptcmd_files then local distcc = instance:extraconf(scriptcmd_files_name, "distcc") - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_files_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) @@ -413,7 +413,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local scriptcmd_file = instance:script(scriptcmd_file_name) if scriptcmd_file then local distcc = instance:extraconf(scriptcmd_file_name, "distcc") - local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_file_name) + local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_file_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) local sourcekind = sourcebatch.sourcekind -- cgit v1.3.1 From e52e6ad4b4ee9cd20aa49a3da87a2c20e88f65f1 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 1 Apr 2025 00:55:39 +0800 Subject: check to add job --- xmake/modules/async/jobgraph.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index b9e9a46da..28126757f 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -89,6 +89,8 @@ function jobgraph:add(name, run, opt) table.insert(groups, job) end end + else + raise("job(%s): has already been added!", name) end end -- cgit v1.3.1 From f2100441c10d29f9932e379077b2b56a29f46389 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 1 Apr 2025 23:29:31 +0800 Subject: fix file job name --- xmake/actions/build/target_utils.lua | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3a1013ef9..3edf9f42b 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -308,6 +308,13 @@ end function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) opt = opt or {} local has_script = false + local job_prefix = target:fullname() + local file_group = sourcebatch.rulename + if target == instance then + job_prefix = job_prefix .. "/target/" .. file_group + else + job_prefix = job_prefix .. "/rule/" .. file_group + end -- call script files if not has_script then @@ -333,7 +340,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) -- target("test") -- on_build_files(function (target, sourcebatch, opt) -- end) - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_files_name) + local jobname = string.format("%s/%s", job_prefix, script_files_name) jobgraph:add(jobname, function (index, total, opt) script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) end) @@ -369,7 +376,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) -- target("test") -- on_build_file(function (target, sourcefile, opt) -- end) - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_file_name) + local jobname = string.format("%s/%s", job_prefix, script_file_name) jobgraph:add(jobname, function (index, total, opt) local sourcekind = sourcebatch.sourcekind for _, sourcefile in ipairs(sourcebatch.sourcefiles) do @@ -392,7 +399,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local scriptcmd_files = instance:script(scriptcmd_files_name) if scriptcmd_files then local distcc = instance:extraconf(scriptcmd_files_name, "distcc") - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_files_name) + local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) @@ -413,7 +420,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local scriptcmd_file = instance:script(scriptcmd_file_name) if scriptcmd_file then local distcc = instance:extraconf(scriptcmd_file_name, "distcc") - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_file_name) + local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) local sourcekind = sourcebatch.sourcekind -- cgit v1.3.1 From 38695f0bf5875dc5db9ccaa8c9ddbf8ef46d4f5e Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 3 Apr 2025 00:47:55 +0800 Subject: add target deps orders --- xmake/actions/build/target_utils.lua | 7 +++++++ xmake/modules/async/jobgraph.lua | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3edf9f42b..9529fd490 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -287,9 +287,16 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) if not targetrefs[targetname] then targetrefs[targetname] = target add_targetjobs(jobgraph, target, opt) + + local linkjob = target:fullname() .. "/link_objects" for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + + local linkjob_dep = dep:fullname() .. "/link_objects" + if jobgraph:has(linkjob) and jobgraph:has(linkjob_dep) then + jobgraph:add_orders(linkjob_dep, linkjob) + end end end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 28126757f..612ff4882 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -107,6 +107,11 @@ function jobgraph:remove(name) end end +-- has the given job or group? +function jobgraph:has(name) + return self._jobs[name] or self._groups[name] +end + -- enter group to add jobs -- -- e.g. -- cgit v1.3.1 From c3632af574ac5f23da61406b49a363f245287b5f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 3 Apr 2025 00:50:38 +0800 Subject: add build.fence --- xmake/actions/build/target_utils.lua | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 9529fd490..a4820fd51 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -283,19 +283,26 @@ end -- add target jobs for the given target and deps function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + local job_kind = opt.job_kind local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target add_targetjobs(jobgraph, target, opt) - local linkjob = target:fullname() .. "/link_objects" + local jobname, jobname_dep for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) - local linkjob_dep = dep:fullname() .. "/link_objects" - if jobgraph:has(linkjob) and jobgraph:has(linkjob_dep) then - jobgraph:add_orders(linkjob_dep, linkjob) + if dep:policy("build.fence") then + jobname = string.format("target/%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("target/%s/end_%s", dep:fullname(), job_kind) + elseif job_kind == "build" then + jobname = target:fullname() .. "/link_objects" + jobname_dep = dep:fullname() .. "/link_objects" + end + if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then + jobgraph:add_orders(jobname_dep, jobname) end end end -- cgit v1.3.1 From c8c97864f3dd431e382548c549957c36c0e60a07 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 3 Apr 2025 23:36:41 +0800 Subject: fix job name --- xmake/actions/build/target_utils.lua | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index a4820fd51..30dd99125 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -116,6 +116,12 @@ end function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false + local job_prefix = target:fullname() + if target == instance then + job_prefix = job_prefix .. "/target" + else + job_prefix = job_prefix .. "/rule/" .. instance:fullname() + end -- call script if not has_script then @@ -139,7 +145,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) -- target("test") -- on_build(function (target, opt) -- end) - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), script_name) + local jobname = string.format("%s/%s", job_prefix, script_name) jobgraph:add(jobname, function (index, total, opt) script(target, {progress = opt.progress}) end) @@ -158,7 +164,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) local scriptcmd_name = opt.scriptcmd_name local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then - local jobname = string.format("%s/%s/%s", target:fullname(), instance:fullname(), scriptcmd_name) + local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = opt.progress}) @@ -294,7 +300,7 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) - if dep:policy("build.fence") then + if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then jobname = string.format("target/%s/begin_%s", target:fullname(), job_kind) jobname_dep = string.format("target/%s/end_%s", dep:fullname(), job_kind) elseif job_kind == "build" then -- cgit v1.3.1 From 00db1966d9dee14ca10abae74ac722680186e0ab Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 4 Apr 2025 00:24:00 +0800 Subject: add file rule --- xmake/actions/build/target_utils.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 30dd99125..266bc3090 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -478,6 +478,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files -- build sourcebatches map + local instances = {target} local sourcebatches_map = {} local sourcebatches_for_target = {} for _, sourcebatch in pairs(sourcebatches) do @@ -496,16 +497,13 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) if ruleinst:script("build_file") or ruleinst:script("build_files") then table.insert(sourcebatches_for_target, sourcebatch) end + table.insert(instances, ruleinst) else table.insert(sourcebatches_for_target, sourcebatch) end end -- TODO sort rules and jobs - local instances = {target} - for _, r in ipairs(target:orderules()) do - table.insert(instances, r) - end -- call target and rules script local jobsize = jobgraph:size() -- cgit v1.3.1 From 9adc2943a3ae362b1713b058b6812dc5954c05f0 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 4 Apr 2025 00:26:16 +0800 Subject: fix link deps --- xmake/actions/build/target_utils.lua | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 266bc3090..3d8b5e361 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -117,9 +117,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false local job_prefix = target:fullname() - if target == instance then - job_prefix = job_prefix .. "/target" - else + if target ~= instance then job_prefix = job_prefix .. "/rule/" .. instance:fullname() end @@ -301,11 +299,11 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then - jobname = string.format("target/%s/begin_%s", target:fullname(), job_kind) - jobname_dep = string.format("target/%s/end_%s", dep:fullname(), job_kind) + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) elseif job_kind == "build" then - jobname = target:fullname() .. "/link_objects" - jobname_dep = dep:fullname() .. "/link_objects" + jobname = target:fullname() .. "/link" + jobname_dep = dep:fullname() .. "/link" end if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then jobgraph:add_orders(jobname_dep, jobname) @@ -331,7 +329,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local job_prefix = target:fullname() local file_group = sourcebatch.rulename if target == instance then - job_prefix = job_prefix .. "/target/" .. file_group + job_prefix = job_prefix .. "/" .. file_group else job_prefix = job_prefix .. "/rule/" .. file_group end -- cgit v1.3.1 From 15d9e01ab37a28d861ce58b1b3701779e94da865 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 4 Apr 2025 00:42:58 +0800 Subject: dump jobgraph --- xmake/actions/build/target_utils.lua | 4 ++-- xmake/core/base/graph.lua | 13 ++++++------ xmake/modules/async/jobgraph.lua | 38 ++++++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3d8b5e361..85da7816e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -181,7 +181,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) local job_kind = opt.job_kind -- the group name, e.g. foo/after_prepare, bar/before_build - local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) + local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) -- the script name, e.g. before/after_prepare, before/after_build local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind @@ -465,7 +465,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local job_kindcmd_files = job_kind .. "cmd_files" -- the group name, e.g. foo/after_prepare_files, bar/before_build_files - local group_name = string.format("%s/%s_%s_files", target:fullname(), stage, job_kind) + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) -- the script name, e.g. before/after_prepare_files, before/after_build_files local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 53f09c59f..fd7675628 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -23,6 +23,7 @@ local table = require("base/table") local queue = require("base/queue") local object = require("base/object") local hashset = require("base/hashset") +local utils = require("base/utils") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -454,15 +455,15 @@ end function graph:dump() local vertices = self:vertices() local edges = self:edges() - print(string.format("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges)) - print("vertices: ") + utils.cprint("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges) + utils.cprint("vertices: ") for _, v in ipairs(vertices) do - print(string.format(" %s", v)) + utils.cprint(" %s", v) end - print("") - print("edges: ") + utils.cprint("") + utils.cprint("edges: ") for _, e in ipairs(edges) do - print(string.format(" %s -> %s", e:from(), e:to())) + utils.cprint(" %s ${color.dump.reference}->${clear} %s", e:from(), e:to()) end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 612ff4882..be1ad57f0 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -142,6 +142,7 @@ end function jobgraph:add_orders(...) local prev local prev_is_group + local prev_name local dag = self._dag local jobs = self._jobs local groups = self._groups @@ -163,16 +164,16 @@ function jobgraph:add_orders(...) assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then if prev_is_group and curr_is_group then - -- we use a fake job as a node to bridge the two groups. - local fakejob = {} + -- we use a bridge job as a node to bridge the two groups. + local bridge = {from_group = prev_name, to_group = name} for _, job in ipairs(prev) do - if not dag:has_edge(job, fakejob) then - dag:add_edge(job, fakejob) + if not dag:has_edge(job, bridge) then + dag:add_edge(job, bridge) end end for _, job in ipairs(curr) do - if not dag:has_edge(fakejob, job) then - dag:add_edge(fakejob, job) + if not dag:has_edge(bridge, job) then + dag:add_edge(bridge, job) end end elseif curr_is_group then @@ -195,6 +196,7 @@ function jobgraph:add_orders(...) end prev = curr prev_is_group = curr_is_group + prev_name = name end end end @@ -226,6 +228,30 @@ function jobgraph:empty() return self:size() == 0 end +-- dump jobgraph +function jobgraph:dump() + print("================================ %s ================================", self) + for _, node in ipairs(self._dag:vertices()) do + debug.setmetatable(node, {__tostring = function (v) + if v.from_group and v.to_group then + return string.format("${dim}bridge<%s, %s>${clear}", v.from_group, v.to_group) + end + return string.format("${color.dump.string_quote}%s${clear}", v.name) + end}) + end + self._dag:dump() + + print("") + print("groups:") + for name, jobs in pairs(self._groups) do + print(" group(%s):", name) + for _, job in ipairs(jobs) do + cprint(" %s", job) + end + end + print("") +end + -- tostring function jobgraph:__tostring() return string.format("", self:name() or "anonymous", self:size()) -- cgit v1.3.1 From c6b6ae7961f8454ab8280ef8d17b5228408096c3 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 4 Apr 2025 00:47:16 +0800 Subject: fix job name --- xmake/actions/build/target_utils.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 85da7816e..36c087548 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -238,8 +238,8 @@ function add_targetjobs(jobgraph, target, opt) end local job_kind = opt.job_kind - local job_begin = string.format("target/%s/begin_%s", target:fullname(), job_kind) - local job_end = string.format("target/%s/end_%s", target:fullname(), job_kind) + local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) + local job_end = string.format("%s/end_%s", target:fullname(), job_kind) jobgraph:add(job_begin, function (index, total, opt) -- enter package environments -- https://github.com/xmake-io/xmake/issues/4033 -- cgit v1.3.1 From 1839ec9aebb573adc36803bf3d632f97410f3666 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 00:50:38 +0800 Subject: sort target rules --- tests/apis/rules_inject_deps/xmake.lua | 6 +----- tests/apis/rules_order/xmake.lua | 20 ++++++++++++++++-- xmake/actions/build/target_utils.lua | 37 ++++++++++++++++++++++++++-------- xmake/core/project/rule.lua | 5 +++++ xmake/modules/async/jobgraph.lua | 2 +- 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/tests/apis/rules_inject_deps/xmake.lua b/tests/apis/rules_inject_deps/xmake.lua index cf73449bb..79b3f8252 100644 --- a/tests/apis/rules_inject_deps/xmake.lua +++ b/tests/apis/rules_inject_deps/xmake.lua @@ -1,10 +1,6 @@ rule("cppfront") set_extensions(".cpp2") - on_load(function (target) - local rule = target:rule("c++.build"):clone() - rule:add("deps", "cppfront", {order = true}) - target:rule_add(rule) - end) + add_buildorders("cppfront", "c++.build") on_build_file(function (target, sourcefile, opt) print("build cppfront file") local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp")) diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua index ea167dca9..8b3eb867a 100644 --- a/tests/apis/rules_order/xmake.lua +++ b/tests/apis/rules_order/xmake.lua @@ -1,7 +1,15 @@ rule("markdown") - add_deps("man", {order = true}) set_extensions(".md", ".markdown") + add_buildorders("man", "markdown") + + before_build(function (target) + print("before_build: markdown") + end) + after_build(function (target) + print("after_build: markdown") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -14,6 +22,14 @@ rule("markdown") rule("man") set_extensions(".man") + + before_build(function (target) + print("before_build: man") + end) + after_build(function (target) + print("after_build: man") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -26,7 +42,7 @@ rule("man") target("test") set_kind("binary") - add_rules("markdown") + add_rules("markdown", "man") add_files("src/*.c") add_files("src/*.md") add_files("src/*.man") diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 36c087548..9ee7a9c0d 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -117,7 +117,9 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false local job_prefix = target:fullname() - if target ~= instance then + if target == instance then + job_prefix = job_prefix .. "/target" + else job_prefix = job_prefix .. "/rule/" .. instance:fullname() end @@ -189,13 +191,11 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - -- TODO sort rules and jobs + -- call target and rules script local instances = {target} for _, r in ipairs(target:orderules()) do table.insert(instances, r) end - - -- call target and rules script local jobsize = jobgraph:size() jobgraph:group(group_name, function () local has_script = false @@ -204,9 +204,12 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do - if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then - has_script = true - end + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then + has_script = true + end + end) -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules if has_script and instance == target and stage == "" then break @@ -219,6 +222,23 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) + -- sort build rules + for _, instance in ipairs(instances) do + local buildorders = table.wrap(instance:get("buildorders")) + for _, buildorder in ipairs(buildorders) do + local joborders = {} + for _, name in ipairs(buildorder) do + local script_group = group_name .. "/" .. name + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) + end + end + end + if jobgraph:size() > jobsize then return group_name end @@ -298,6 +318,7 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + -- build.across_targets_in_parallel is deprecated if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then jobname = string.format("%s/begin_%s", target:fullname(), job_kind) jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) @@ -329,7 +350,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local job_prefix = target:fullname() local file_group = sourcebatch.rulename if target == instance then - job_prefix = job_prefix .. "/" .. file_group + job_prefix = job_prefix .. "/target/" .. file_group else job_prefix = job_prefix .. "/rule/" .. file_group end diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index 63f954580..a2fd661e8 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -280,6 +280,11 @@ function rule.apis() , "rule.add_deps" , "rule.add_imports" } + , groups = + { + -- rule.add_xxx + "rule.add_buildorders" + } , script = { -- rule.on_xxx diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index be1ad57f0..4bbc53ff4 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -109,7 +109,7 @@ end -- has the given job or group? function jobgraph:has(name) - return self._jobs[name] or self._groups[name] + return (self._jobs[name] or self._groups[name]) ~= nil end -- enter group to add jobs -- cgit v1.3.1 From e6a81c6a5c84842d6f5b54cfd6d4cdd97f45eef9 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 18:35:59 +0800 Subject: rename to add_orders --- tests/apis/rules_inject_deps/xmake.lua | 2 +- tests/apis/rules_order/xmake.lua | 2 +- xmake/actions/build/target_utils.lua | 35 +++++++++++++++------------------- xmake/core/project/rule.lua | 2 +- xmake/modules/private/utils/rule.lua | 34 +++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 23 deletions(-) create mode 100644 xmake/modules/private/utils/rule.lua diff --git a/tests/apis/rules_inject_deps/xmake.lua b/tests/apis/rules_inject_deps/xmake.lua index 79b3f8252..880bcde32 100644 --- a/tests/apis/rules_inject_deps/xmake.lua +++ b/tests/apis/rules_inject_deps/xmake.lua @@ -1,6 +1,6 @@ rule("cppfront") set_extensions(".cpp2") - add_buildorders("cppfront", "c++.build") + add_orders("cppfront", "c++.build") on_build_file(function (target, sourcefile, opt) print("build cppfront file") local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp")) diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua index 8b3eb867a..e05dc7bd2 100644 --- a/tests/apis/rules_order/xmake.lua +++ b/tests/apis/rules_order/xmake.lua @@ -1,7 +1,7 @@ rule("markdown") set_extensions(".md", ".markdown") - add_buildorders("man", "markdown") + add_orders("man", "markdown") before_build(function (target) print("before_build: markdown") diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 9ee7a9c0d..4c33b5adf 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,14 +27,7 @@ import("core.project.project") import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") - --- get rule --- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target -function _get_rule(target, rulename) - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - return ruleinst -end +import("private.utils.rule", {alias = "rule_utils"}) -- clean target for rebuilding function _clean_target(target) @@ -222,19 +215,21 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) - -- sort build rules + -- sort build rules, TODO deps for _, instance in ipairs(instances) do - local buildorders = table.wrap(instance:get("buildorders")) - for _, buildorder in ipairs(buildorders) do - local joborders = {} - for _, name in ipairs(buildorder) do - local script_group = group_name .. "/" .. name - if jobgraph:has(script_group) then - table.insert(joborders, script_group) + local orders = table.wrap(instance:get("orders")) + if #orders > 0 then + for _, order in ipairs(orders) do + local joborders = {} + for _, rulename in ipairs(order) do + local script_group = group_name .. "/" .. rulename + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) end - end - if #joborders > 0 then - jobgraph:add_orders(joborders) end end end @@ -503,7 +498,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, sourcebatch in pairs(sourcebatches) do local rulename = sourcebatch.rulename if rulename then - local ruleinst = _get_rule(target, rulename) + local ruleinst = rule_utils.get_rule(target, rulename) sourcebatches_map[ruleinst] = sourcebatch -- avoid duplicate scripts being called twice in the target, -- we just build sourcebatch with on_build_files scripts diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index a2fd661e8..ccab257c7 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -283,7 +283,7 @@ function rule.apis() , groups = { -- rule.add_xxx - "rule.add_buildorders" + "rule.add_orders" } , script = { diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua new file mode 100644 index 000000000..19706ad04 --- /dev/null +++ b/xmake/modules/private/utils/rule.lua @@ -0,0 +1,34 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file rule.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") + +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + -- cgit v1.3.1 From 529f59f911ddba8a2ef015f50df7a9582dd2bdb4 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 22:07:22 +0800 Subject: sort file rules --- xmake/actions/build/target_utils.lua | 64 ++++++++++++++++-------------------- xmake/modules/private/utils/rule.lua | 36 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 4c33b5adf..3d71ff55e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -215,28 +215,14 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) - -- sort build rules, TODO deps - for _, instance in ipairs(instances) do - local orders = table.wrap(instance:get("orders")) - if #orders > 0 then - for _, order in ipairs(orders) do - local joborders = {} - for _, rulename in ipairs(order) do - local script_group = group_name .. "/" .. rulename - if jobgraph:has(script_group) then - table.insert(joborders, script_group) - end - end - if #joborders > 0 then - jobgraph:add_orders(joborders) - end - end - end + -- no any new jobs + if jobgraph:size() == jobsize then + return end - if jobgraph:size() > jobsize then - return group_name - end + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name end -- add target jobs for the given target @@ -517,8 +503,6 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) end end - -- TODO sort rules and jobs - -- call target and rules script local jobsize = jobgraph:size() jobgraph:group(group_name, function () @@ -530,26 +514,34 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) } local has_target_script = false for _, instance in ipairs(instances) do - if instance == target then - for _, sourcebatch in ipairs(sourcebatches_for_target) do - local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules - if has_script and stage == "" then - has_target_script = true + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if instance == target then + for _, sourcebatch in ipairs(sourcebatches_for_target) do + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end + end + elseif not has_target_script then -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch then + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) end end - elseif not has_target_script then -- rule - local sourcebatch = sourcebatches_map[instance] - if sourcebatch then - add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - end - end + end) end end) - if jobgraph:size() > jobsize then - return group_name + -- no any new jobs + if jobgraph:size() == jobsize then + return end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name end -- add file jobs for the given target diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua index 19706ad04..531486f61 100644 --- a/xmake/modules/private/utils/rule.lua +++ b/xmake/modules/private/utils/rule.lua @@ -32,3 +32,39 @@ function get_rule(target, rulename) return ruleinst end +-- build rules orders in jobgraph, we need to add rule job with groups +-- +-- like this: +-- @code +-- local root_group = "" +-- for _, ruleinst in ipairs(rules) do +-- local script_group = root_group .. "/" .. ruleinst:fullname() +-- jobgraph:group(script_group, function () +-- jobgraph:add("xxx", function (index, total, opt) +-- -- call rule script +-- end) +-- end) +-- end +-- +function build_orders_in_jobgraph(jobgraph, rules, opt) + opt = opt or {} + local root_group = assert(opt.root_group) + for _, ruleinst in ipairs(rules) do + local orders = table.wrap(ruleinst:get("orders")) + if #orders > 0 then + for _, order in ipairs(orders) do + local joborders = {} + for _, rulename in ipairs(order) do + local script_group = root_group .. "/" .. rulename + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) + end + end + end + end +end + -- cgit v1.3.1 From ca185a3cc835182a9c290bc53bfec505abd60833 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 22:15:27 +0800 Subject: update order opt --- xmake/rules/cppfront/xmake.lua | 11 ++--------- xmake/rules/lex_yacc/lex/xmake.lua | 2 +- xmake/rules/qt/moc/xmake.lua | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/xmake/rules/cppfront/xmake.lua b/xmake/rules/cppfront/xmake.lua index 22663f184..772c59487 100644 --- a/xmake/rules/cppfront/xmake.lua +++ b/xmake/rules/cppfront/xmake.lua @@ -22,7 +22,6 @@ rule("cppfront.build.h2") set_extensions(".h2") on_buildcmd_file(function (target, batchcmds, sourcefile_h2, opt) - -- get cppfront import("lib.detect.find_tool") local cppfront = assert(find_tool("cppfront", {check = "-h"}), "cppfront not found!") @@ -42,12 +41,11 @@ rule("cppfront.build.h2") batchcmds:set_depcache(target:dependfile(sourcefile_h)) end) --- define rule: cppfront.build rule("cppfront.build.cpp2") set_extensions(".cpp2") -- .h2 must compile before .cpp2 - add_deps("cppfront.build.h2", {order = true}) + add_orders("cppfront.build.h2", "cppfront.build.cpp2") on_load(function (target) -- only cppfront source files? we need to patch cxx source kind for linker @@ -64,7 +62,6 @@ rule("cppfront.build.cpp2") end end) on_buildcmd_file(function (target, batchcmds, sourcefile_cpp2, opt) - -- get cppfront import("lib.detect.find_tool") local cppfront = assert(find_tool("cppfront", {check = "-h"}), "cppfront not found!") @@ -84,6 +81,7 @@ rule("cppfront.build.cpp2") batchcmds:add_depfiles(path.join(root_dir, match_h2)) end end + -- add commands local argv = {"-o", path(sourcefile_cpp), path(sourcefile_cpp2)} batchcmds:show_progress(opt.progress, "${color.build.object}compiling.cpp2 %s", sourcefile_cpp2) @@ -97,14 +95,9 @@ rule("cppfront.build.cpp2") batchcmds:set_depcache(target:dependfile(objectfile)) end) - -- define rule: cppfront rule("cppfront") - - -- add_build.h2 rules add_deps("cppfront.build.h2") - - -- add build rules add_deps("cppfront.build.cpp2") -- set compiler runtime, e.g. vs runtime diff --git a/xmake/rules/lex_yacc/lex/xmake.lua b/xmake/rules/lex_yacc/lex/xmake.lua index 3ee16b7f4..d57f06036 100644 --- a/xmake/rules/lex_yacc/lex/xmake.lua +++ b/xmake/rules/lex_yacc/lex/xmake.lua @@ -21,7 +21,7 @@ -- define rule: lex rule("lex") add_deps("c++") - add_deps("yacc", {order = true}) + add_orders("yacc", "lex") set_extensions(".l", ".ll") before_buildcmd_file(function (target, batchcmds, sourcefile_lex, opt) diff --git a/xmake/rules/qt/moc/xmake.lua b/xmake/rules/qt/moc/xmake.lua index 2807f2bc9..4ece876cf 100644 --- a/xmake/rules/qt/moc/xmake.lua +++ b/xmake/rules/qt/moc/xmake.lua @@ -20,7 +20,7 @@ rule("qt.moc") add_deps("qt.env") - add_deps("qt.ui", {order = true}) + add_orders("qt.ui", "qt.moc") set_extensions(".h", ".hpp") before_buildcmd_file(function (target, batchcmds, sourcefile, opt) import("core.tool.compiler") -- cgit v1.3.1 From 8f6a3130eff588abda27b22fa86e936b6500f50c Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 23:30:36 +0800 Subject: fix headerunit --- .../rules/c++/modules/modules_support/builder.lua | 49 ++++++++++++++++------ .../c++/modules/modules_support/gcc/builder.lua | 2 - xmake/rules/c++/modules/xmake.lua | 11 +---- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 0fd9920b1..7d5ebd0c8 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -45,11 +45,15 @@ function _build_modules(target, sourcebatch, modules, opt) cppfile = cppfile or module.cppfile local deps = {} - for _, dep in ipairs(table.keys(module.requires or {})) do + for name, req in pairs(module.requires or {}) do + -- we need to use the full path as dep name if requre item is headerunit + local dep = name + if req.method:startswith("include-") and req.path then + dep = path.normalize(req.path) + end local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep table.insert(deps, depname) end - opt.build_module(deps, module, name, objectfile, cppfile) ::continue:: @@ -58,7 +62,6 @@ end -- build target headerunits function _build_headerunits(target, headerunits, opt) - local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true}) if opt.stl_headerunit then outputdir = path.join(outputdir, "stl") @@ -72,11 +75,9 @@ function _build_headerunits(target, headerunits, opt) local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) local key = path.normalize(headerunit.path) local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) - if build then mark_build(target, key) end - opt.build_headerunit(headerunit, key, bmifile, outputdir, build) end end @@ -292,6 +293,7 @@ end -- build modules for jobgraph function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) local jobdeps = {} + local jobsize = jobgraph:size() local build_modules_group = target:fullname() .. "/module/build_modules" jobgraph:group(build_modules_group, function () @@ -313,10 +315,8 @@ function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) end}) ) end) - for jobname, deps in pairs(jobdeps) do - for _, depname in ipairs(deps) do - jobgraph:add_orders(depname, jobname) - end + if jobgraph:size() > jobsize then + return build_modules_group, jobdeps end end @@ -385,7 +385,7 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local build_modules_group = target:fullname() .. "/module/build_modules" + local jobsize = jobgraph:size() local build_headerunits_group = target:fullname() .. "/module/build_headerunits" jobgraph:group(build_headerunits_group, function () local build_headerunits = function(headerunits) @@ -393,7 +393,7 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) local job_name = target:fullname() .. "/module/" .. key - _builder(target).make_headerunit_buildjobs(target, + _builder(target).make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) end })) @@ -409,12 +409,13 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, build_headerunits(user_headerunits) end end) - jobgraph:add_orders(build_headerunits_group, build_modules_group) + if jobgraph:size() > jobsize then + return build_headerunits_group + end end -- build headerunits for batchcmds function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) if not user_headerunits and not stl_headerunits then return @@ -441,6 +442,28 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end +-- build modules and headerunits, and we need to build headerunits first +function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) + if jobgraph.add_orders then + local build_modules_group, jobdeps = build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local build_headerunits_group = build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + if build_modules_group then + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end + if build_headerunits_group then + jobgraph:add_orders(build_headerunits_group, build_modules_group) + end + end + else -- deprecated + build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + end +end + +-- generate metadata function generate_metadata(target, modules) local public_modules for _, module in table.orderpairs(modules) do diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 05e8380da..4033eefd5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -361,7 +361,6 @@ end -- build headerunit file for batchjobs function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local _headerunit = headerunit _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) @@ -443,7 +442,6 @@ end -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 42eab0c19..5f17e2099 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -112,15 +112,8 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build modules and headerunits, and we need to build headerunits first - if jobgraph.add_orders then - builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - else -- deprecated - builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - end - + -- build modules and headerunits + builder.build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else sourcebatch.objectfiles = {} -- cgit v1.3.1 From 365603a10bc127abb5f7c94b67959b2665745c72 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 00:04:31 +0800 Subject: fix build rust --- xmake/actions/build/target_utils.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3d71ff55e..92723d46c 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -306,6 +306,12 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) elseif job_kind == "build" then jobname = target:fullname() .. "/link" jobname_dep = dep:fullname() .. "/link" + if not jobgraph:has(jobname) then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + end + if not jobgraph:has(jobname_dep) then + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + end end if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then jobgraph:add_orders(jobname_dep, jobname) -- cgit v1.3.1 From 406b9d08d56eae4893082d75e84f016f4619f66c Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 13:14:56 +0800 Subject: fix framework and bundle links --- xmake/rules/xcode/bundle/xmake.lua | 2 +- xmake/rules/xcode/framework/xmake.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/xcode/bundle/xmake.lua b/xmake/rules/xcode/bundle/xmake.lua index 31c0a8296..972d7d75e 100644 --- a/xmake/rules/xcode/bundle/xmake.lua +++ b/xmake/rules/xcode/bundle/xmake.lua @@ -62,7 +62,7 @@ rule("xcode.bundle") end end) - after_build(function (target, opt) + after_link(function (target, opt) -- imports import("core.base.option") diff --git a/xmake/rules/xcode/framework/xmake.lua b/xmake/rules/xcode/framework/xmake.lua index 10b2f337f..46850c019 100644 --- a/xmake/rules/xcode/framework/xmake.lua +++ b/xmake/rules/xcode/framework/xmake.lua @@ -78,7 +78,7 @@ rule("xcode.framework") end end) - after_build(function (target, opt) + after_link(function (target, opt) -- imports import("core.base.option") -- cgit v1.3.1 From 99f68b5c58d822170c84e2b6c0b3f7f974e009be Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 20:58:25 +0800 Subject: remove unused prepare jobs --- .../rules/c++/modules/modules_support/builder.lua | 46 +++++++++++- .../modules/modules_support/compiler_support.lua | 14 ---- .../modules/modules_support/dependency_scanner.lua | 60 ++++++++-------- xmake/rules/c++/modules/xmake.lua | 84 ++++++---------------- 4 files changed, 96 insertions(+), 108 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 7d5ebd0c8..2a0977524 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -457,7 +457,10 @@ function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, o jobgraph:add_orders(build_headerunits_group, build_modules_group) end end - else -- deprecated + elseif jobgraph.runcmds then + build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + elseif jobgraph.newjob then -- deprecated build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) end @@ -571,3 +574,44 @@ function is_dependencies_changed(target, module) end return requires, changed end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch, opt) + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + end + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) + end + end + + -- patch objectfiles and dependencies + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + + local dependfile = target:dependfile(sourcefile or objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index a5a32e2c5..eed30a320 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -69,20 +69,6 @@ function strip_flags(target, flags) return _compiler_support(target).strip_flags(target, flags) end --- patch sourcebatch -function patch_sourcebatch(target, sourcebatch) - sourcebatch.sourcekind = "cxx" - sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - table.insert(sourcebatch.objectfiles, objectfile) - - local dependfile = target:dependfile(sourcefile or objectfile) - table.insert(sourcebatch.dependfiles, dependfile) - end -end - -- get bmi extension function get_bmi_extension(target) return _compiler_support(target).get_bmi_extension() diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 4a8f08909..94b8a4804 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -202,9 +202,8 @@ function _get_edges(nodes, modules) return edges end -function _get_package_modules(target, package, opt) +function _get_package_modules(target, package) local package_modules - local modulesdir = path.join(package:installdir(), "modules") local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) for _, metafile in ipairs(metafiles) do @@ -213,42 +212,39 @@ function _get_package_modules(target, package, opt) local moduleonly = not package:libraryfiles() package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata, external = {moduleonly = moduleonly}} end - return package_modules end --- generate dependency files -function _generate_dependencies(target, sourcebatch, opt) - local changed = false - if opt.jobgraph then - local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/module_dependency_scanner", function(index) - local sourcefile = sourcebatch.sourcefiles[index] - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end, {comax = jobs, total = #sourcebatch.sourcefiles}) - else - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end - end - return changed -end --- get module dependencies -function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:fullname() .. "/" .. sourcebatch.rulename - local modules = compiler_support.memcache():get2("modules", cachekey) - if modules == nil then - modules = compiler_support.localcache():get2("modules", cachekey) - opt.progress = opt.progress or 0 - local changed = _generate_dependencies(target, sourcebatch, opt) - if changed or modules == nil then +-- generate module dependencies +function generate_module_dependencies(target, jobgraph, sourcebatch, opt) + local parsejob = target:fullname() .. "/parse_module_dependencies" + jobgraph:add(parsejob, function (index, total, opt) + local changed = compiler_support.memcache():get2("modules", "dependencies_changed") + if changed then + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) - modules = _parse_dependencies_data(target, moduleinfos) + local modules = _parse_dependencies_data(target, moduleinfos) compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() end - compiler_support.memcache():set2("modules", cachekey, modules) + end) + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local jobname = target:fullname() .. "/generate_module_dependencies/" .. sourcefile + jobgraph:add(jobname, function (index, total, opt) + local changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) + if changed then + compiler_support.memcache():set2("modules", "dependencies_changed", true) + end + end) + jobgraph:add_orders(jobname, parsejob) end +end + +-- get module dependencies +function get_module_dependencies(target, sourcebatch) + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename + local modules = compiler_support.localcache():get2("modules", cachekey) + assert(modules, "no module dependencies!") return modules end @@ -387,7 +383,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end -- extract packages modules dependencies -function get_all_packages_modules(target, opt) +function get_all_packages_modules(target) -- parse all meta-info and append their informations to the package store local packages = target:pkgs() or {} @@ -397,7 +393,7 @@ function get_all_packages_modules(target, opt) local packages_modules for _, package in table.orderpairs(packages) do - local package_modules = _get_package_modules(target, package, opt) + local package_modules = _get_package_modules(target, package) if package_modules then packages_modules = packages_modules or {} table.join2(packages_modules, package_modules) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 5f17e2099..87429f90c 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -70,6 +70,20 @@ rule("c++.build.modules.builder") set_sourcekinds("cxx") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") + -- generate module dependencies + on_prepare_files(function (target, jobgraph, sourcebatch, opt) + if target:data("cxx.has_modules") then + import("modules_support.builder") + import("modules_support.dependency_scanner") + + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + + -- generate module dependencies + dependency_scanner.generate_module_dependencies(target, jobgraph, sourcebatch, opt) + end + end, {jobgraph = true}) + -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, jobgraph, sourcebatch, opt) if target:data("cxx.has_modules") then @@ -77,36 +91,11 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end - - opt = opt or {} - opt.jobgraph = true - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) @@ -134,45 +123,18 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end - - opt = opt or {} - opt.jobgraph = false - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles -- build headerunits and modules - builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - + builder.build_modules_and_headerunits(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else -- avoid duplicate linking of object files of non-module programs -- cgit v1.3.1 From da540028213b73ca014bc7aa0f7938a12e344ee8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 21:09:49 +0800 Subject: fix generate deps --- xmake/rules/c++/modules/modules_support/builder.lua | 20 ++++++++++---------- xmake/rules/c++/modules/xmake.lua | 6 ------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 2a0977524..7ec120cc9 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -34,7 +34,6 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles - local jobgraph = opt.jobgraph for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then @@ -51,7 +50,7 @@ function _build_modules(target, sourcebatch, modules, opt) if req.method:startswith("include-") and req.path then dep = path.normalize(req.path) end - local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep + local depname = target:fullname() .. "/module/" .. dep table.insert(deps, depname) end opt.build_module(deps, module, name, objectfile, cppfile) @@ -279,11 +278,11 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/module/" .. (name or cppfile) - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, - {module = module, objectfile = objectfile, cppfile = cppfile}) - end + build_module = function(deps, module, name, objectfile, cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, + {module = module, objectfile = objectfile, cppfile = cppfile}) + end })) -- build batchjobs for modules @@ -330,9 +329,10 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, module, _, objectfile, cppfile) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) - end + build_module = function(_, module, _, objectfile, cppfile) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, { + module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + end })) batchcmds:set_depmtime(depmtime) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 87429f90c..d0c0e0387 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -91,9 +91,6 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- patch sourcebatch - builder.patch_sourcebatch(target, sourcebatch) - -- get module dependencies local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then @@ -123,9 +120,6 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- patch sourcebatch - builder.patch_sourcebatch(target, sourcebatch) - -- get module dependencies local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- cgit v1.3.1 From 54227eb4e653a46b8eb9ad9642d7641db052288b Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 21:38:39 +0800 Subject: fix progress --- xmake/actions/build/build.lua | 2 ++ xmake/actions/build/build_files.lua | 2 ++ xmake/actions/build/target_utils.lua | 4 ++-- xmake/modules/async/runjobs.lua | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 22097669a..d3fdd8e80 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -30,6 +30,7 @@ import("deprecated.build", {alias = "deprecated_build"}) function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" + opt.progress_factor = 0.05 target_utils.run_targetjobs(targets_root, opt) end @@ -37,6 +38,7 @@ end function _build(targets_root, opt) opt = opt or {} opt.job_kind = "build" + opt.progress_factor = 0.95 if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index e26c11aae..854f7de3f 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -75,6 +75,7 @@ end function _prepare_files(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" + opt.progress_factor = 0.05 opt.filepatterns = _get_file_patterns(opt.sourcefiles) target_utils.run_filejobs(targets_root, opt) end @@ -83,6 +84,7 @@ end function _build_files(targets_root, opt) opt = opt or {} opt.job_kind = "build" + opt.progress_factor = 0.95 opt.filepatterns = _get_file_patterns(opt.sourcefiles) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 92723d46c..147c25187 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -686,7 +686,7 @@ function run_targetjobs(targets_root, opt) if errors and progress.showing_without_scroll() then print("") end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) os.cd(curdir) return true end @@ -704,7 +704,7 @@ function run_filejobs(targets_root, opt) if errors and progress.showing_without_scroll() then print("") end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) os.cd(curdir) return true end diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index d2af72843..adf68018d 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -163,6 +163,7 @@ function main(name, jobs, opt) local abort_errors local progress_wrapper = {} local job_pending + local progress_factor = opt.progress_factor or 1.0 progress_wrapper.current = function () return count end @@ -171,7 +172,7 @@ function main(name, jobs, opt) end progress_wrapper.percent = function () if total and total > 0 then - return math.floor((count * 100) / total) + return math.floor((count * progress_factor * 100) / total) else return 0 end -- cgit v1.3.1 From 3b2e44ef7fce4baca2ede46bf1edb8cbf42734bb Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 21:45:39 +0800 Subject: fix bin2c order --- xmake/rules/utils/bin2c/xmake.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/rules/utils/bin2c/xmake.lua b/xmake/rules/utils/bin2c/xmake.lua index 5d0dbfd31..392b2f20a 100644 --- a/xmake/rules/utils/bin2c/xmake.lua +++ b/xmake/rules/utils/bin2c/xmake.lua @@ -20,6 +20,7 @@ rule("utils.bin2c") set_extensions(".bin") + add_orders("utils.bin2c", "c++.build.modules.builder") on_load(function (target) local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c") if not os.isdir(headerdir) then @@ -27,7 +28,7 @@ rule("utils.bin2c") end target:add("includedirs", headerdir) end) - before_buildcmd_file(function (target, batchcmds, sourcefile_bin, opt) + on_preparecmd_file(function (target, batchcmds, sourcefile_bin, opt) -- get header file local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c") -- cgit v1.3.1 From 43c3b8607775f7ee5b52e24a7f785b4c7a5ab292 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 22:03:58 +0800 Subject: add modules with pch test --- tests/projects/c++/modules/hello_with_pch/src/hello.mpp | 10 ++++++++++ tests/projects/c++/modules/hello_with_pch/src/main.cpp | 8 ++++++++ tests/projects/c++/modules/hello_with_pch/src/test.h | 1 + tests/projects/c++/modules/hello_with_pch/test.lua | 1 + tests/projects/c++/modules/hello_with_pch/xmake.lua | 7 +++++++ xmake/rules/c++/precompiled_header/xmake.lua | 6 ++++-- 6 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/projects/c++/modules/hello_with_pch/src/hello.mpp create mode 100644 tests/projects/c++/modules/hello_with_pch/src/main.cpp create mode 100644 tests/projects/c++/modules/hello_with_pch/src/test.h create mode 100644 tests/projects/c++/modules/hello_with_pch/test.lua create mode 100644 tests/projects/c++/modules/hello_with_pch/xmake.lua diff --git a/tests/projects/c++/modules/hello_with_pch/src/hello.mpp b/tests/projects/c++/modules/hello_with_pch/src/hello.mpp new file mode 100644 index 000000000..124bd72bc --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/hello.mpp @@ -0,0 +1,10 @@ +module; +#include + +export module hello; + +export namespace hello { + void say(const char* str) { + printf("%s\n", str); + } +} diff --git a/tests/projects/c++/modules/hello_with_pch/src/main.cpp b/tests/projects/c++/modules/hello_with_pch/src/main.cpp new file mode 100644 index 000000000..739309dd4 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/main.cpp @@ -0,0 +1,8 @@ +#include "test.h" + +import hello; + +int main() { + hello::say("hello module!"); + return 0; +} diff --git a/tests/projects/c++/modules/hello_with_pch/src/test.h b/tests/projects/c++/modules/hello_with_pch/src/test.h new file mode 100644 index 000000000..d5a732287 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/test.h @@ -0,0 +1 @@ +#include diff --git a/tests/projects/c++/modules/hello_with_pch/test.lua b/tests/projects/c++/modules/hello_with_pch/test.lua new file mode 100644 index 000000000..7717f8049 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/test.lua @@ -0,0 +1 @@ +inherit(".test_base") diff --git a/tests/projects/c++/modules/hello_with_pch/xmake.lua b/tests/projects/c++/modules/hello_with_pch/xmake.lua new file mode 100644 index 000000000..79ddb6b58 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/xmake.lua @@ -0,0 +1,7 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +target("hello") + set_kind("binary") + set_pcxxheader("src/test.h") + add_files("src/*.cpp", "src/*.mpp") diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 7216a1f13..3f30c1ea1 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -19,18 +19,20 @@ -- rule("c.build.pcheader") + add_orders("c.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) - before_build(function (target, opt) + on_prepare(function (target, opt) import("private.action.build.pcheader").build(target, "c", opt) end) rule("c++.build.pcheader") + add_orders("c++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) - before_build(function (target, opt) + on_prepare(function (target, opt) import("private.action.build.pcheader").build(target, "cxx", opt) end) -- cgit v1.3.1 From 6d8671865bbe9026305186c54b4ef82ebb80a4bc Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 22:11:00 +0800 Subject: enable jobgraph for pch --- xmake/modules/private/action/build/object.lua | 1 + xmake/modules/private/action/build/pcheader.lua | 6 +++--- xmake/rules/c++/precompiled_header/xmake.lua | 12 ++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 0028b4b6f..a8b66aa54 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -174,6 +174,7 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) end function main(target, jobgraph, sourcebatch, opt) + opt = opt or {} if jobgraph.add_orders then _add_jobgraph(target, jobgraph, sourcebatch, opt) else diff --git a/xmake/modules/private/action/build/pcheader.lua b/xmake/modules/private/action/build/pcheader.lua index 20e857d7f..ec06c8fb8 100644 --- a/xmake/modules/private/action/build/pcheader.lua +++ b/xmake/modules/private/action/build/pcheader.lua @@ -20,7 +20,7 @@ -- imports import("core.language.language") -import("object") +import("object", {alias = "build_objects"}) function config(target, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) @@ -55,7 +55,7 @@ function config(target, langkind, opt) end -- add batch jobs to build the precompiled header file -function build(target, langkind, opt) +function build(target, jobgraph, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) if pcheaderfile then local sourcefile = pcheaderfile @@ -63,6 +63,6 @@ function build(target, langkind, opt) local dependfile = target:dependfile(objectfile) local sourcekind = language.langkinds()[langkind] local sourcebatch = {sourcekind = sourcekind, sourcefiles = {sourcefile}, objectfiles = {objectfile}, dependfiles = {dependfile}} - object.build(target, sourcebatch, opt) + build_objects(target, jobgraph, sourcebatch, opt) end end diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 3f30c1ea1..08ecf3fd7 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -23,16 +23,16 @@ rule("c.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) - on_prepare(function (target, opt) - import("private.action.build.pcheader").build(target, "c", opt) - end) + on_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "c", opt) + end, {jobgraph = true}) rule("c++.build.pcheader") add_orders("c++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) - on_prepare(function (target, opt) - import("private.action.build.pcheader").build(target, "cxx", opt) - end) + on_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "cxx", opt) + end, {jobgraph = true}) -- cgit v1.3.1 From d88716e932f28b3546b90a71e79ca9d8d5fb6551 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 22:37:25 +0800 Subject: fix pch --- xmake/rules/c++/precompiled_header/xmake.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 08ecf3fd7..c5fbd15a7 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -23,7 +23,7 @@ rule("c.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) - on_prepare(function (target, jobgraph, opt) + before_prepare(function (target, jobgraph, opt) import("private.action.build.pcheader").build(target, jobgraph, "c", opt) end, {jobgraph = true}) @@ -32,7 +32,7 @@ rule("c++.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) - on_prepare(function (target, jobgraph, opt) + before_prepare(function (target, jobgraph, opt) import("private.action.build.pcheader").build(target, jobgraph, "cxx", opt) end, {jobgraph = true}) -- cgit v1.3.1 From 162f55410e2a1ed2a81dddfc4189bb85158f2525 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 22:44:20 +0800 Subject: fix pcheader for objc --- xmake/rules/c++/precompiled_header/xmake.lua | 1 - xmake/rules/objc++/precompiled_header/xmake.lua | 13 +++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index c5fbd15a7..30ab31106 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -19,7 +19,6 @@ -- rule("c.build.pcheader") - add_orders("c.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) diff --git a/xmake/rules/objc++/precompiled_header/xmake.lua b/xmake/rules/objc++/precompiled_header/xmake.lua index db2d9f036..9f2b21ade 100644 --- a/xmake/rules/objc++/precompiled_header/xmake.lua +++ b/xmake/rules/objc++/precompiled_header/xmake.lua @@ -22,15 +22,16 @@ rule("objc.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "m", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "m", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "m", opt) + end, {jobgraph = true}) rule("objc++.build.pcheader") + add_orders("objc++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "mxx", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "mxx", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "mxx", opt) + end, {jobgraph = true}) -- cgit v1.3.1 From b2d2727fdcc1d955b4095c57b6d02ff665bcbfd9 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 23:02:59 +0800 Subject: disable module and pch for gcc --- tests/projects/c++/modules/hello_with_pch/test.lua | 2 +- tests/projects/c++/modules/test_pch.lua | 31 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/projects/c++/modules/test_pch.lua diff --git a/tests/projects/c++/modules/hello_with_pch/test.lua b/tests/projects/c++/modules/hello_with_pch/test.lua index 7717f8049..16ef46834 100644 --- a/tests/projects/c++/modules/hello_with_pch/test.lua +++ b/tests/projects/c++/modules/hello_with_pch/test.lua @@ -1 +1 @@ -inherit(".test_base") +inherit(".test_pch") diff --git a/tests/projects/c++/modules/test_pch.lua b/tests/projects/c++/modules/test_pch.lua new file mode 100644 index 000000000..98286c229 --- /dev/null +++ b/tests/projects/c++/modules/test_pch.lua @@ -0,0 +1,31 @@ +import("lib.detect.find_tool") +import("core.base.semver") +import("detect.sdks.find_vstudio") +import("utils.ci.is_running", {alias = "ci_is_running"}) + +function _build() + if ci_is_running() then + os.run("xmake -rvD") + else + os.run("xmake -r") + end + local outdata = os.iorun("xmake") + if outdata then + if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then + raise("Modules incremental compilation does not work\n%s", outdata) + end + end +end + +function main(t) + -- TODO c++ modules with pch does not work for gcc now. + if is_host("linux") then + local clang = find_tool("clang", {version = true}) + if clang then + os.exec("xmake f --toolchain=clang -c --yes --policies=build.c++.modules.std:n,build.c++.clang.fallbackscanner") + _build() + end + else + _build() + end +end -- cgit v1.3.1 From 72870df34c9a3278fe4a92fa5591f6738f80ada6 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 23:16:28 +0800 Subject: Delete tests/projects/c++/modules/hello_with_pch/test.lua --- tests/projects/c++/modules/hello_with_pch/test.lua | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/projects/c++/modules/hello_with_pch/test.lua diff --git a/tests/projects/c++/modules/hello_with_pch/test.lua b/tests/projects/c++/modules/hello_with_pch/test.lua deleted file mode 100644 index 16ef46834..000000000 --- a/tests/projects/c++/modules/hello_with_pch/test.lua +++ /dev/null @@ -1 +0,0 @@ -inherit(".test_pch") -- cgit v1.3.1 From c091d7b4d8f13ed4b17d23df8867ec30e8003c4c Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 7 Apr 2025 22:36:14 +0800 Subject: remove across_targets_in_parallel policy --- tests/projects/other/build_deps/xmake.lua | 6 +++--- tests/projects/other/merge_archive/xmake.lua | 3 ++- tests/projects/other/merge_object/xmake.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 2 +- xmake/rules/fortran/xmake.lua | 2 +- xmake/rules/go/xmake.lua | 2 +- xmake/rules/vala/xmake.lua | 2 +- xmake/rules/xcode/application/xmake.lua | 13 ------------- 8 files changed, 10 insertions(+), 22 deletions(-) diff --git a/tests/projects/other/build_deps/xmake.lua b/tests/projects/other/build_deps/xmake.lua index 5729b5c8f..b2324e3dd 100644 --- a/tests/projects/other/build_deps/xmake.lua +++ b/tests/projects/other/build_deps/xmake.lua @@ -4,7 +4,7 @@ target("dep1") set_kind("static") add_deps("dep3") add_files("src/dep1.c") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep3"):targetfile()) @@ -21,7 +21,7 @@ target("dep2") set_kind("static") add_deps("dep3") add_files("src/dep2.c") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep3"):targetfile()) @@ -38,7 +38,7 @@ target("dep3") set_kind("static") add_files("src/dep3.c") add_deps("dep4", "dep5") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep4"):targetfile()) diff --git a/tests/projects/other/merge_archive/xmake.lua b/tests/projects/other/merge_archive/xmake.lua index 11092dec9..7cf74471d 100644 --- a/tests/projects/other/merge_archive/xmake.lua +++ b/tests/projects/other/merge_archive/xmake.lua @@ -3,18 +3,19 @@ add_rules("mode.debug", "mode.release") target("add") set_kind("static") add_files("src/add.c") + set_policy("build.fence", true) set_targetdir("$(buildir)/merge_archive") target("sub") set_kind("static") add_files("src/sub.c") + set_policy("build.fence", true) set_targetdir("$(buildir)/merge_archive") target("mul") set_kind("static") add_deps("add", "sub") add_files("src/mul.c") - set_policy("build.across_targets_in_parallel", false) if is_plat("windows") then add_files("$(buildir)/merge_archive/*.lib") else diff --git a/tests/projects/other/merge_object/xmake.lua b/tests/projects/other/merge_object/xmake.lua index b8799a0b8..bbd7c1f98 100644 --- a/tests/projects/other/merge_object/xmake.lua +++ b/tests/projects/other/merge_object/xmake.lua @@ -1,9 +1,9 @@ add_rules("mode.debug", "mode.release") -set_policy("build.across_targets_in_parallel", false) target("merge_object") set_kind("static") add_files("src/interface.c") + set_policy("build.fence", true) after_build_file(function (target, sourcefile) os.cp(target:objectfile(sourcefile), "$(buildir)/merge_object/") end) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index d0c0e0387..a27633182 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -37,7 +37,7 @@ rule("c++.build.modules") -- even if some sub-targets do not contain C++ modules. -- -- maybe we will have a more fine-grained configuration strategy to disable it in the future. - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- disable ccache for this target -- diff --git a/xmake/rules/fortran/xmake.lua b/xmake/rules/fortran/xmake.lua index 4ab11906d..d10993a91 100644 --- a/xmake/rules/fortran/xmake.lua +++ b/xmake/rules/fortran/xmake.lua @@ -35,7 +35,7 @@ rule("fortran.build") add_deps("fortran.build.modules") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) end) on_build_files(function (target, sourcebatch, opt) import("private.action.build.object").build(target, sourcebatch, opt) diff --git a/xmake/rules/go/xmake.lua b/xmake/rules/go/xmake.lua index 58890e260..fd4cdfb4a 100644 --- a/xmake/rules/go/xmake.lua +++ b/xmake/rules/go/xmake.lua @@ -23,7 +23,7 @@ rule("go.build") add_deps("go.env") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- xxx.a if target:is_static() then target:set("prefixname", "") diff --git a/xmake/rules/vala/xmake.lua b/xmake/rules/vala/xmake.lua index caebab9d8..e2262a9ec 100644 --- a/xmake/rules/vala/xmake.lua +++ b/xmake/rules/vala/xmake.lua @@ -26,7 +26,7 @@ rule("vala.build") set_sourcekinds("cc") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- get vapi file local vapifile = target:data("vala.vapifile") diff --git a/xmake/rules/xcode/application/xmake.lua b/xmake/rules/xcode/application/xmake.lua index b4969b3df..7d5edc668 100644 --- a/xmake/rules/xcode/application/xmake.lua +++ b/xmake/rules/xcode/application/xmake.lua @@ -27,19 +27,6 @@ rule("xcode.application") -- we must set kind before target.on_load(), may we will use target in on_load() on_load("load") - -- depend xcode.framework? we need to disable `build.across_targets_in_parallel` policy - after_load(function (target) - local across_targets_in_parallel - for _, dep in ipairs(target:orderdeps()) do - if dep:rule("xcode.framework") then - across_targets_in_parallel = false - end - end - if across_targets_in_parallel ~= nil then - target:set("policy", "build.across_targets_in_parallel", across_targets_in_parallel) - end - end) - -- build *.app after_build("build") -- cgit v1.3.1 From 279801b485afbb433aac44c60c79379fb99f5153 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 7 Apr 2025 22:36:40 +0800 Subject: add warings tip --- xmake/actions/build/target_utils.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 147c25187..50190ec57 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -303,6 +303,9 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then jobname = string.format("%s/begin_%s", target:fullname(), job_kind) jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + if dep:policy("build.across_targets_in_parallel") == false then + wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") + end elseif job_kind == "build" then jobname = target:fullname() .. "/link" jobname_dep = dep:fullname() .. "/link" -- cgit v1.3.1 From 3ebc4d126a43600bb33bf1a4db7ccc27e07457b9 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 7 Apr 2025 22:37:18 +0800 Subject: improve build.fence wrap --- xmake/actions/build/target_utils.lua | 51 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 50190ec57..2f68830c0 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -92,6 +92,32 @@ function _match_sourcebatches(target, filepatterns) end end +-- add targetjobs and deps orders +function _add_targetjobs_orders(jobgraph, target, dep, opt) + local jobname, jobname_dep + local job_kind = opt.job_kind + if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + -- build.across_targets_in_parallel is deprecated + if dep:policy("build.across_targets_in_parallel") == false then + wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") + end + elseif job_kind == "build" then + jobname = target:fullname() .. "/link" + jobname_dep = dep:fullname() .. "/link" + if not jobgraph:has(jobname) then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + end + if not jobgraph:has(jobname_dep) then + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + end + end + if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then + jobgraph:add_orders(jobname_dep, jobname) + end +end + -- add target jobs for the builtin script function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then @@ -288,37 +314,14 @@ end -- add target jobs for the given target and deps function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) - local job_kind = opt.job_kind local targetname = target:fullname() if not targetrefs[targetname] then targetrefs[targetname] = target add_targetjobs(jobgraph, target, opt) - - local jobname, jobname_dep for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) - - -- build.across_targets_in_parallel is deprecated - if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - if dep:policy("build.across_targets_in_parallel") == false then - wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") - end - elseif job_kind == "build" then - jobname = target:fullname() .. "/link" - jobname_dep = dep:fullname() .. "/link" - if not jobgraph:has(jobname) then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - end - if not jobgraph:has(jobname_dep) then - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - end - end - if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then - jobgraph:add_orders(jobname_dep, jobname) - end + _add_targetjobs_orders(jobgraph, target, dep, opt) end end end -- cgit v1.3.1 From 07ac1c481e0a30161389328365efcbebc15d88d9 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 7 Apr 2025 22:40:03 +0800 Subject: fix distcc --- xmake/modules/async/jobgraph.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 4bbc53ff4..1ed5873b7 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -73,7 +73,7 @@ function jobgraph:add(name, run, opt) local dag = self._dag local jobs = self._jobs if not jobs[name] then - local job = {name = name, run = run, opt = opt} + local job = {name = name, run = run, distcc = opt.distcc} jobs[name] = job dag:add_vertex(job) self._size = self._size + 1 -- cgit v1.3.1 From 6fc28a78bf95490b5c230a42c0f4a40ab5aad2e4 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:40:15 +0800 Subject: move target files --- xmake/actions/build/build.lua | 2 +- xmake/actions/build/build_files.lua | 2 +- xmake/actions/build/target_utils.lua | 717 -------------------- .../modules/private/action/build/build_binary.lua | 38 ++ .../private/action/build/build_moduleonly.lua | 26 + .../modules/private/action/build/build_object.lua | 26 + .../modules/private/action/build/build_shared.lua | 26 + .../modules/private/action/build/build_static.lua | 26 + .../modules/private/action/build/link_objects.lua | 68 ++ .../modules/private/action/build/prepare_files.lua | 27 + xmake/modules/private/action/build/target.lua | 718 +++++++++++++++++++++ 11 files changed, 957 insertions(+), 719 deletions(-) delete mode 100644 xmake/actions/build/target_utils.lua create mode 100644 xmake/modules/private/action/build/build_binary.lua create mode 100644 xmake/modules/private/action/build/build_moduleonly.lua create mode 100644 xmake/modules/private/action/build/build_object.lua create mode 100644 xmake/modules/private/action/build/build_shared.lua create mode 100644 xmake/modules/private/action/build/build_static.lua create mode 100644 xmake/modules/private/action/build/link_objects.lua create mode 100644 xmake/modules/private/action/build/prepare_files.lua create mode 100644 xmake/modules/private/action/build/target.lua diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index d3fdd8e80..270b24ea7 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -22,8 +22,8 @@ import("core.base.option") import("core.project.config") import("core.project.project") -import("target_utils") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_utils"}) import("deprecated.build", {alias = "deprecated_build"}) -- run prepare jobs diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 854f7de3f..12d9b00ee 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -23,8 +23,8 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") -import("target_utils") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_utils"}) import("deprecated.build_files", {alias = "deprecated_build_files"}) -- convert all sourcefiles to lua pattern diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua deleted file mode 100644 index 2f68830c0..000000000 --- a/xmake/actions/build/target_utils.lua +++ /dev/null @@ -1,717 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file target_utils.lua --- - --- imports -import("core.base.option") -import("core.base.hashset") -import("core.project.rule") -import("core.project.config") -import("core.project.project") -import("async.runjobs", {alias = "async_runjobs"}) -import("async.jobgraph", {alias = "async_jobgraph"}) -import("private.utils.batchcmds") -import("private.utils.rule", {alias = "rule_utils"}) - --- clean target for rebuilding -function _clean_target(target) - if target:targetfile() then - os.tryrm(target:symbolfile()) - os.tryrm(target:targetfile()) - end -end - --- match source files -function _match_sourcefiles(sourcefile, filepatterns) - for _, filepattern in ipairs(filepatterns) do - if sourcefile:match(filepattern.pattern) == sourcefile then - if filepattern.excludes then - if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then - sourcefile = sourcefile:sub(#filepattern.rootdir + 2) - end - for _, exclude in ipairs(filepattern.excludes) do - if sourcefile:match(exclude) == sourcefile then - return false - end - end - end - return true - end - end -end - --- match sourcebatches -function _match_sourcebatches(target, filepatterns) - local newbatches = {} - local sourcecount = 0 - for rulename, sourcebatch in pairs(target:sourcebatches()) do - local objectfiles = sourcebatch.objectfiles - local dependfiles = sourcebatch.dependfiles - local sourcekind = sourcebatch.sourcekind - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do - if _match_sourcefiles(sourcefile, filepatterns) then - local newbatch = newbatches[rulename] - if not newbatch then - newbatch = {} - newbatch.sourcekind = sourcekind - newbatch.rulename = rulename - newbatch.sourcefiles = {} - end - table.insert(newbatch.sourcefiles, sourcefile) - if objectfiles then - newbatch.objectfiles = newbatch.objectfiles or {} - table.insert(newbatch.objectfiles, objectfiles[idx]) - end - if dependfiles then - newbatch.dependfiles = newbatch.dependfiles or {} - table.insert(newbatch.dependfiles, dependfiles[idx]) - end - newbatches[rulename] = newbatch - sourcecount = sourcecount + 1 - end - end - end - if sourcecount > 0 then - return newbatches - end -end - --- add targetjobs and deps orders -function _add_targetjobs_orders(jobgraph, target, dep, opt) - local jobname, jobname_dep - local job_kind = opt.job_kind - if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - -- build.across_targets_in_parallel is deprecated - if dep:policy("build.across_targets_in_parallel") == false then - wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") - end - elseif job_kind == "build" then - jobname = target:fullname() .. "/link" - jobname_dep = dep:fullname() .. "/link" - if not jobgraph:has(jobname) then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - end - if not jobgraph:has(jobname_dep) then - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - end - end - if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then - jobgraph:add_orders(jobname_dep, jobname) - end -end - --- add target jobs for the builtin script -function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) - if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then - if job_kind == "prepare" then - import("builtin.prepare_files", {anonymous = true})(jobgraph, target) - elseif job_kind == "link" then - import("builtin.link_objects", {anonymous = true})(jobgraph, target) - else - import("builtin.build_" .. target:kind(), {anonymous = true})(jobgraph, target) - end - end -end - --- add target jobs for the given script -function add_targetjobs_for_script(jobgraph, target, instance, opt) - opt = opt or {} - local has_script = false - local job_prefix = target:fullname() - if target == instance then - job_prefix = job_prefix .. "/target" - else - job_prefix = job_prefix .. "/rule/" .. instance:fullname() - end - - -- call script - if not has_script then - local script_name = opt.script_name - local script = instance:script(script_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build(function (target, jobgraph, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then - script(target, jobgraph) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = opt.progress}) - end) - end - has_script = true - end - end - - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, opt) - -- end) - if not has_script then - local scriptcmd_name = opt.scriptcmd_name - local scriptcmd = instance:script(scriptcmd_name) - if scriptcmd then - local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - return has_script -end - --- add target jobs with the given stage --- stage: before, after or "" -function add_targetjobs_with_stage(jobgraph, target, stage, opt) - opt = opt or {} - local job_kind = opt.job_kind - - -- the group name, e.g. foo/after_prepare, bar/before_build - local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) - - -- the script name, e.g. before/after_prepare, before/after_build - local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind - - -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd - local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - - -- call target and rules script - local instances = {target} - for _, r in ipairs(target:orderules()) do - table.insert(instances, r) - end - local jobsize = jobgraph:size() - jobgraph:group(group_name, function () - local has_script = false - local script_opt = { - script_name = script_name, - scriptcmd_name = scriptcmd_name - } - for _, instance in ipairs(instances) do - local script_group = group_name .. "/" .. instance:fullname() - jobgraph:group(script_group, function () - if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then - has_script = true - end - end) - -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules - if has_script and instance == target and stage == "" then - break - end - end - - -- call builtin script, e.g. on_prepare, on_build, ... - if not has_script and stage == "" then - add_targetjobs_for_builtin_script(jobgraph, target, job_kind) - end - end) - - -- no any new jobs - if jobgraph:size() == jobsize then - return - end - - -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) - return group_name -end - --- add target jobs for the given target -function add_targetjobs(jobgraph, target, opt) - opt = opt or {} - if not target:is_enabled() then - return - end - - local pkgenvs = _g.pkgenvs - if pkgenvs == nil then - pkgenvs = {} - _g.pkgenvs = pkgenvs - end - - local job_kind = opt.job_kind - local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) - local job_end = string.format("%s/end_%s", target:fullname(), job_kind) - jobgraph:add(job_begin, function (index, total, opt) - -- enter package environments - -- https://github.com/xmake-io/xmake/issues/4033 - -- - -- maybe mixing envs isn't a great solution, - -- but it's the most efficient compromise compared to setting envs in every on_build_file. - -- - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = target:pkgenvs() - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - -- clean target first if rebuild - if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then - _clean_target(target) - end - end) - - jobgraph:add(job_end, function (index, total, opt) - -- restore environments - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = nil - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - end) - - -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) - jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) -end - --- add target jobs for the given target and deps -function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) - local targetname = target:fullname() - if not targetrefs[targetname] then - targetrefs[targetname] = target - add_targetjobs(jobgraph, target, opt) - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) - _add_targetjobs_orders(jobgraph, target, dep, opt) - end - end -end - --- get target jobs -function get_targetjobs(targets_root, opt) - local jobgraph = async_jobgraph.new(opt.job_kind) - local targetrefs = {} - for _, target in ipairs(targets_root) do - add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) - end - return jobgraph -end - --- add file jobs for the given script -function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) - opt = opt or {} - local has_script = false - local job_prefix = target:fullname() - local file_group = sourcebatch.rulename - if target == instance then - job_prefix = job_prefix .. "/target/" .. file_group - else - job_prefix = job_prefix .. "/rule/" .. file_group - end - - -- call script files - if not has_script then - local script_files_name = opt.script_files_name - local script_files = instance:script(script_files_name) - if script_files then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build_files(function (target, jobgraph, sourcebatch, opt) - -- end, {jobgraph = true}) - local distcc = instance:extraconf(script_files_name, "distcc") - if instance:extraconf(script_files_name, "jobgraph") then - script_files(target, jobgraph, sourcebatch, {distcc = distcc}) - elseif instance:extraconf(script_files_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", - instance:fullname(), script_files_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build_files(function (target, sourcebatch, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_files_name) - jobgraph:add(jobname, function (index, total, opt) - script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) - end) - end - has_script = true - end - end - - -- call script file - if not has_script then - local script_file_name = opt.script_file_name - local script_file = instance:script(script_file_name) - if script_file then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build_file(function (target, jobgraph, sourcefile, opt) - -- end, {jobgraph = true}) - local distcc = instance:extraconf(script_file_name, "distcc") - if instance:extraconf(script_file_name, "jobgraph") then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) - end - elseif instance:extraconf(script_file_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", - instance:fullname(), script_file_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build_file(function (target, sourcefile, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_file_name) - jobgraph:add(jobname, function (index, total, opt) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) - end - end) - end - has_script = true - end - end - - -- call command script files - -- e.g. - -- - -- target("test") - -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) - -- end) - if not has_script then - local scriptcmd_files_name = opt.scriptcmd_files_name - local scriptcmd_files = instance:script(scriptcmd_files_name) - if scriptcmd_files then - local distcc = instance:extraconf(scriptcmd_files_name, "distcc") - local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - - -- call command script file - -- e.g. - -- - -- target("test") - -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) - -- end) - if not has_script then - local scriptcmd_file_name = opt.scriptcmd_file_name - local scriptcmd_file = instance:script(scriptcmd_file_name) - if scriptcmd_file then - local distcc = instance:extraconf(scriptcmd_file_name, "distcc") - local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) - end - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - return has_script -end - --- add file jobs with the given stage --- stage: before, after or "" --- -function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) - opt = opt or {} - local job_kind = opt.job_kind - local job_kind_file = job_kind .. "_file" - local job_kind_files = job_kind .. "_files" - local job_kindcmd_file = job_kind .. "cmd_file" - local job_kindcmd_files = job_kind .. "cmd_files" - - -- the group name, e.g. foo/after_prepare_files, bar/before_build_files - local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) - - -- the script name, e.g. before/after_prepare_files, before/after_build_files - local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file - local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files - - -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files - local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file - local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files - - -- build sourcebatches map - local instances = {target} - local sourcebatches_map = {} - local sourcebatches_for_target = {} - for _, sourcebatch in pairs(sourcebatches) do - local rulename = sourcebatch.rulename - if rulename then - local ruleinst = rule_utils.get_rule(target, rulename) - sourcebatches_map[ruleinst] = sourcebatch - -- avoid duplicate scripts being called twice in the target, - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - if ruleinst:script("build_file") or ruleinst:script("build_files") then - table.insert(sourcebatches_for_target, sourcebatch) - end - table.insert(instances, ruleinst) - else - table.insert(sourcebatches_for_target, sourcebatch) - end - end - - -- call target and rules script - local jobsize = jobgraph:size() - jobgraph:group(group_name, function () - local script_opt = { - script_file_name = script_file_name, - script_files_name = script_files_name, - scriptcmd_file_name = scriptcmd_file_name, - scriptcmd_files_name = scriptcmd_files_name - } - local has_target_script = false - for _, instance in ipairs(instances) do - local script_group = group_name .. "/" .. instance:fullname() - jobgraph:group(script_group, function () - if instance == target then - for _, sourcebatch in ipairs(sourcebatches_for_target) do - local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules - if has_script and stage == "" then - has_target_script = true - end - end - elseif not has_target_script then -- rule - local sourcebatch = sourcebatches_map[instance] - if sourcebatch then - add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - end - end - end) - end - end) - - -- no any new jobs - if jobgraph:size() == jobsize then - return - end - - -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) - return group_name -end - --- add file jobs for the given target -function add_filejobs(jobgraph, target, opt) - opt = opt or {} - if not target:is_enabled() then - return - end - - -- get sourcebatches - local sourcebatches - local filepatterns = opt.filepatterns - if filepatterns then - sourcebatches = _match_sourcebatches(target, filepatterns) - else - sourcebatches = target:sourcebatches() - end - - -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) - local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) - local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) - jobgraph:add_orders(group_before, group, group_after) -end - --- add file jobs for the given target and deps -function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) - local targetname = target:fullname() - if not targetrefs[targetname] then - targetrefs[targetname] = target - add_filejobs(jobgraph, target, opt) - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) - end - end -end - --- get files jobs -function get_filejobs(targets_root, opt) - local jobgraph = async_jobgraph.new(opt.job_kind) - local targetrefs = {} - for _, target in ipairs(targets_root) do - add_filejobs_and_deps(jobgraph, target, targetrefs, opt) - end - return jobgraph -end - --- add link jobs for the given target -function add_linkjobs(jobgraph, target, opt) - opt = opt or {} - opt.job_kind = "link" - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) - jobgraph:add_orders(group_before, group, group_after) -end - --- get link depfiles -function get_linkdepfiles(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles -end - --- get all root targets -function get_root_targets(targetnames, opt) - opt = opt or {} - - -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end - return targets_root -end - --- run target-level jobs, e.g. on_prepare, on_build, ... -function run_targetjobs(targets_root, opt) - opt = opt or {} - local job_kind = opt.job_kind - local jobgraph = get_targetjobs(targets_root, opt) - if jobgraph and not jobgraph:empty() then - local curdir = os.curdir() - async_runjobs(job_kind, jobgraph, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) - os.cd(curdir) - return true - end -end - --- run files-level jobs, e.g. on_prepare_files, on_build_files, ... -function run_filejobs(targets_root, opt) - opt = opt or {} - local job_kind = opt.job_kind - local jobgraph = get_filejobs(targets_root, opt) - if jobgraph and not jobgraph:empty() then - local curdir = os.curdir() - async_runjobs(job_kind, jobgraph, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) - os.cd(curdir) - return true - end -end diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua new file mode 100644 index 000000000..c30addfef --- /dev/null +++ b/xmake/modules/private/action/build/build_binary.lua @@ -0,0 +1,38 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_binary.lua +-- + +-- imports +import("build_object") +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + local objects_group = target:fullname() .. "/objects" + local jobsize = jobgraph:size() + jobgraph:group(objects_group, function () + build_object(jobgraph, target) + end) + if jobgraph:size() > jobsize then + local link_group = target:fullname() .. "/link" + jobgraph:group(link_group, function () + target_utils.add_linkjobs(jobgraph, target) + end) + jobgraph:add_orders(objects_group, link_group) + end +end diff --git a/xmake/modules/private/action/build/build_moduleonly.lua b/xmake/modules/private/action/build/build_moduleonly.lua new file mode 100644 index 000000000..e3cc87c27 --- /dev/null +++ b/xmake/modules/private/action/build/build_moduleonly.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_moduleonly.lua +-- + +-- imports +import("build_object") + +function main(jobgraph, target) + build_object(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua new file mode 100644 index 000000000..2a44bcea6 --- /dev/null +++ b/xmake/modules/private/action/build/build_object.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_object.lua +-- + +-- imports +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) +end diff --git a/xmake/modules/private/action/build/build_shared.lua b/xmake/modules/private/action/build/build_shared.lua new file mode 100644 index 000000000..077489710 --- /dev/null +++ b/xmake/modules/private/action/build/build_shared.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_shared.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target) + build_binary(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/build_static.lua b/xmake/modules/private/action/build/build_static.lua new file mode 100644 index 000000000..8b9b2ffaf --- /dev/null +++ b/xmake/modules/private/action/build/build_static.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_static.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target) + build_binary(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua new file mode 100644 index 000000000..099674d3e --- /dev/null +++ b/xmake/modules/private/action/build/link_objects.lua @@ -0,0 +1,68 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file link_objects.lua +-- + +-- imports +import("core.base.option") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("build_object") +import("private.action.build.target", {alias = "target_utils"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = target_utils.get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +function main(jobgraph, target) + local linkjob = target:fullname() .. "/link_objects" + jobgraph:add(linkjob, function (index, total, opt) + _do_link_target(target, opt) + end) +end diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua new file mode 100644 index 000000000..a0ecc242a --- /dev/null +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -0,0 +1,27 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_files.lua +-- + +-- imports +import("core.base.option") +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) +end diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua new file mode 100644 index 000000000..b6d428692 --- /dev/null +++ b/xmake/modules/private/action/build/target.lua @@ -0,0 +1,718 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file target.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.rule") +import("core.project.config") +import("core.project.project") +import("async.runjobs", {alias = "async_runjobs"}) +import("async.jobgraph", {alias = "async_jobgraph"}) +import("private.utils.batchcmds") +import("private.utils.rule", {alias = "rule_utils"}) + +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- match sourcebatches +function _match_sourcebatches(target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return newbatches + end +end + +-- add targetjobs and deps orders +function _add_targetjobs_orders(jobgraph, target, dep, opt) + local jobname, jobname_dep + local job_kind = opt.job_kind + if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + -- build.across_targets_in_parallel is deprecated + if dep:policy("build.across_targets_in_parallel") == false then + wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") + end + elseif job_kind == "build" then + jobname = target:fullname() .. "/link" + jobname_dep = dep:fullname() .. "/link" + if not jobgraph:has(jobname) then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + end + if not jobgraph:has(jobname_dep) then + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + end + end + if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then + jobgraph:add_orders(jobname_dep, jobname) + end +end + +-- add target jobs for the builtin script +function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then + if job_kind == "prepare" then + import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target) + elseif job_kind == "link" then + import("private.action.build.link_objects", {anonymous = true})(jobgraph, target) + else + import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target) + end + end +end + +-- add target jobs for the given script +function add_targetjobs_for_script(jobgraph, target, instance, opt) + opt = opt or {} + local has_script = false + local job_prefix = target:fullname() + if target == instance then + job_prefix = job_prefix .. "/target" + else + job_prefix = job_prefix .. "/rule/" .. instance:fullname() + end + + -- call script + if not has_script then + local script_name = opt.script_name + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true + end + end + + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + if not has_script then + local scriptcmd_name = opt.scriptcmd_name + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add target jobs with the given stage +-- stage: before, after or "" +function add_targetjobs_with_stage(jobgraph, target, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + + -- the group name, e.g. foo/after_prepare, bar/before_build + local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare, before/after_build + local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind + + -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd + local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") + + -- call target and rules script + local instances = {target} + for _, r in ipairs(target:orderules()) do + table.insert(instances, r) + end + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local has_script = false + local script_opt = { + script_name = script_name, + scriptcmd_name = scriptcmd_name + } + for _, instance in ipairs(instances) do + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then + has_script = true + end + end) + -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules + if has_script and instance == target and stage == "" then + break + end + end + + -- call builtin script, e.g. on_prepare, on_build, ... + if not has_script and stage == "" then + add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name +end + +-- add target jobs for the given target +function add_targetjobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + local pkgenvs = _g.pkgenvs + if pkgenvs == nil then + pkgenvs = {} + _g.pkgenvs = pkgenvs + end + + local job_kind = opt.job_kind + local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) + local job_end = string.format("%s/end_%s", target:fullname(), job_kind) + jobgraph:add(job_begin, function (index, total, opt) + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target first if rebuild + if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + end) + + jobgraph:add(job_end, function (index, total, opt) + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + end) + + -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) +end + +-- add target jobs for the given target and deps +function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_targetjobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + _add_targetjobs_orders(jobgraph, target, dep, opt) + end + end +end + +-- get target jobs +function get_targetjobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add file jobs for the given script +function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) + opt = opt or {} + local has_script = false + local job_prefix = target:fullname() + local file_group = sourcebatch.rulename + if target == instance then + job_prefix = job_prefix .. "/target/" .. file_group + else + job_prefix = job_prefix .. "/rule/" .. file_group + end + + -- call script files + if not has_script then + local script_files_name = opt.script_files_name + local script_files = instance:script(script_files_name) + if script_files then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, jobgraph, sourcebatch, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_files_name, "distcc") + if instance:extraconf(script_files_name, "jobgraph") then + script_files(target, jobgraph, sourcebatch, {distcc = distcc}) + elseif instance:extraconf(script_files_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_files_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, sourcebatch, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_files_name) + jobgraph:add(jobname, function (index, total, opt) + script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) + end) + end + has_script = true + end + end + + -- call script file + if not has_script then + local script_file_name = opt.script_file_name + local script_file = instance:script(script_file_name) + if script_file then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, jobgraph, sourcefile, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_file_name, "distcc") + if instance:extraconf(script_file_name, "jobgraph") then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) + end + elseif instance:extraconf(script_file_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_file_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, sourcefile, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_file_name) + jobgraph:add(jobname, function (index, total, opt) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + end) + end + has_script = true + end + end + + -- call command script files + -- e.g. + -- + -- target("test") + -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- end) + if not has_script then + local scriptcmd_files_name = opt.scriptcmd_files_name + local scriptcmd_files = instance:script(scriptcmd_files_name) + if scriptcmd_files then + local distcc = instance:extraconf(scriptcmd_files_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + + -- call command script file + -- e.g. + -- + -- target("test") + -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) + -- end) + if not has_script then + local scriptcmd_file_name = opt.scriptcmd_file_name + local scriptcmd_file = instance:script(scriptcmd_file_name) + if scriptcmd_file then + local distcc = instance:extraconf(scriptcmd_file_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add file jobs with the given stage +-- stage: before, after or "" +-- +function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + local job_kind_file = job_kind .. "_file" + local job_kind_files = job_kind .. "_files" + local job_kindcmd_file = job_kind .. "cmd_file" + local job_kindcmd_files = job_kind .. "cmd_files" + + -- the group name, e.g. foo/after_prepare_files, bar/before_build_files + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare_files, before/after_build_files + local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file + local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files + + -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files + local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file + local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files + + -- build sourcebatches map + local instances = {target} + local sourcebatches_map = {} + local sourcebatches_for_target = {} + for _, sourcebatch in pairs(sourcebatches) do + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = rule_utils.get_rule(target, rulename) + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + table.insert(instances, ruleinst) + else + table.insert(sourcebatches_for_target, sourcebatch) + end + end + + -- call target and rules script + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local script_opt = { + script_file_name = script_file_name, + script_files_name = script_files_name, + scriptcmd_file_name = scriptcmd_file_name, + scriptcmd_files_name = scriptcmd_files_name + } + local has_target_script = false + for _, instance in ipairs(instances) do + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if instance == target then + for _, sourcebatch in ipairs(sourcebatches_for_target) do + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end + end + elseif not has_target_script then -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch then + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + end + end + end) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name +end + +-- add file jobs for the given target +function add_filejobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + -- get sourcebatches + local sourcebatches + local filepatterns = opt.filepatterns + if filepatterns then + sourcebatches = _match_sourcebatches(target, filepatterns) + else + sourcebatches = target:sourcebatches() + end + + -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files + local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + jobgraph:add_orders(group_before, group, group_after) +end + +-- add file jobs for the given target and deps +function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_filejobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) + end + end +end + +-- get files jobs +function get_filejobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add link jobs for the given target +function add_linkjobs(jobgraph, target, opt) + opt = opt or {} + opt.job_kind = "link" + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + jobgraph:add_orders(group_before, group, group_after) +end + +-- get link depfiles +function get_linkdepfiles(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end + +-- get all root targets +function get_root_targets(targetnames, opt) + opt = opt or {} + + -- get root targets + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + return targets_root +end + +-- run target-level jobs, e.g. on_prepare, on_build, ... +function run_targetjobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_targetjobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + +-- run files-level jobs, e.g. on_prepare_files, on_build_files, ... +function run_filejobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_filejobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + -- cgit v1.3.1 From f72d519dfe7b3a3744660ed15c620dbd5344c16c Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:41:42 +0800 Subject: prepare targets in cmakelists generator --- xmake/actions/build/build.lua | 8 ++++---- xmake/actions/build/build_files.lua | 8 ++++---- xmake/modules/private/action/build/build_binary.lua | 4 ++-- xmake/modules/private/action/build/build_object.lua | 4 ++-- xmake/modules/private/action/build/link_objects.lua | 4 ++-- xmake/modules/private/action/build/prepare_files.lua | 4 ++-- xmake/plugins/project/cmake/cmakelists.lua | 19 ++++++++----------- 7 files changed, 24 insertions(+), 27 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 270b24ea7..d4f5658fb 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -23,7 +23,7 @@ import("core.base.option") import("core.project.config") import("core.project.project") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("deprecated.build", {alias = "deprecated_build"}) -- run prepare jobs @@ -31,7 +31,7 @@ function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" opt.progress_factor = 0.05 - target_utils.run_targetjobs(targets_root, opt) + target_buildutils.run_targetjobs(targets_root, opt) end -- run build jobs @@ -42,13 +42,13 @@ function _build(targets_root, opt) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end - target_utils.run_targetjobs(targets_root, opt) + target_buildutils.run_targetjobs(targets_root, opt) end function main(targetnames, opt) -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) + local targets_root = target_buildutils.get_root_targets(targetnames, opt) -- prepare to build _prepare(targets_root, opt) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 12d9b00ee..1bbad79f6 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -24,7 +24,7 @@ import("core.base.hashset") import("core.project.config") import("core.project.project") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("deprecated.build_files", {alias = "deprecated_build_files"}) -- convert all sourcefiles to lua pattern @@ -77,7 +77,7 @@ function _prepare_files(targets_root, opt) opt.job_kind = "prepare" opt.progress_factor = 0.05 opt.filepatterns = _get_file_patterns(opt.sourcefiles) - target_utils.run_filejobs(targets_root, opt) + target_buildutils.run_filejobs(targets_root, opt) end -- run build files jobs @@ -89,7 +89,7 @@ function _build_files(targets_root, opt) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end - if not target_utils.run_filejobs(targets_root, opt) then + if not target_buildutils.run_filejobs(targets_root, opt) then wprint("%s not found!", opt.sourcefiles) end end @@ -97,7 +97,7 @@ end function main(targetnames, opt) -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) + local targets_root = target_buildutils.get_root_targets(targetnames, opt) -- prepare to build files _prepare_files(targets_root, opt) diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua index c30addfef..72263e6c8 100644 --- a/xmake/modules/private/action/build/build_binary.lua +++ b/xmake/modules/private/action/build/build_binary.lua @@ -20,7 +20,7 @@ -- imports import("build_object") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) local objects_group = target:fullname() .. "/objects" @@ -31,7 +31,7 @@ function main(jobgraph, target) if jobgraph:size() > jobsize then local link_group = target:fullname() .. "/link" jobgraph:group(link_group, function () - target_utils.add_linkjobs(jobgraph, target) + target_buildutils.add_linkjobs(jobgraph, target) end) jobgraph:add_orders(objects_group, link_group) end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua index 2a44bcea6..2a6baa59a 100644 --- a/xmake/modules/private/action/build/build_object.lua +++ b/xmake/modules/private/action/build/build_object.lua @@ -19,8 +19,8 @@ -- -- imports -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) + target_buildutils.add_filejobs(jobgraph, target, {job_kind = "build"}) end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua index 099674d3e..25214d27e 100644 --- a/xmake/modules/private/action/build/link_objects.lua +++ b/xmake/modules/private/action/build/link_objects.lua @@ -25,7 +25,7 @@ import("core.tool.compiler") import("core.project.depend") import("utils.progress") import("build_object") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -33,7 +33,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua index a0ecc242a..d8ca351ca 100644 --- a/xmake/modules/private/action/build/prepare_files.lua +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -20,8 +20,8 @@ -- imports import("core.base.option") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) + target_buildutils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 22edabcd2..464101268 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -31,6 +31,7 @@ import("lib.detect.find_tool") import("private.utils.batchcmds") import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -298,6 +299,12 @@ function _get_flags_from_target(target, flagkind) return results end +-- prepare targets +function _prepare_targets() + local targets_root = target_buildutils.get_root_targets() + target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) +end + -- set compiler function _set_compiler(cmakelists) if config.get("toolchain") then @@ -1331,21 +1338,11 @@ function _generate_cmakelists(cmakelists, outputdir) end end --- make function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) - - -- open the cmakelists local cmakelists = io.open(path.join(outputdir, "CMakeLists.txt"), "w") - - -- generate cmakelists + _prepare_targets() _generate_cmakelists(cmakelists, outputdir) - - -- close the cmakelists cmakelists:close() - - -- leave project directory os.cd(oldir) end -- cgit v1.3.1 From abd58c9c0e5043d73b127de9cd387e416bd5090f Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:44:33 +0800 Subject: prepare targets for more generators --- xmake/plugins/project/clang/compile_commands.lua | 1 + xmake/plugins/project/clang/compile_flags.lua | 6 +++--- xmake/plugins/project/cmake/cmakelists.lua | 9 +-------- xmake/plugins/project/make/makefile.lua | 7 +++---- xmake/plugins/project/ninja/build_ninja.lua | 8 ++++---- xmake/plugins/project/utils/target_cmds.lua | 7 +++++++ xmake/plugins/project/vstudio/impl/vs201x.lua | 13 ++++--------- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/xmake/plugins/project/clang/compile_commands.lua b/xmake/plugins/project/clang/compile_commands.lua index 46a9bfa64..c36cb8ade 100644 --- a/xmake/plugins/project/clang/compile_commands.lua +++ b/xmake/plugins/project/clang/compile_commands.lua @@ -317,6 +317,7 @@ end function make(outputdir) local oldir = os.cd(os.projectdir()) local jsonfile = io.open(path.join(outputdir, "compile_commands.json"), "w") + target_cmds.prepare_targets() _add_targets(jsonfile) jsonfile:close() os.cd(oldir) diff --git a/xmake/plugins/project/clang/compile_flags.lua b/xmake/plugins/project/clang/compile_flags.lua index 3c76b1bdb..5cfedbb19 100644 --- a/xmake/plugins/project/clang/compile_flags.lua +++ b/xmake/plugins/project/clang/compile_flags.lua @@ -85,10 +85,11 @@ end -- - https://clang.llvm.org/docs/JSONCompilationDatabase.html -- function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- make all local flags = {} flags = _make_all(flags) @@ -100,6 +101,5 @@ function make(outputdir) end flagfile:close() - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 464101268..f826cc453 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -31,7 +31,6 @@ import("lib.detect.find_tool") import("private.utils.batchcmds") import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) -import("private.action.build.target", {alias = "target_buildutils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -299,12 +298,6 @@ function _get_flags_from_target(target, flagkind) return results end --- prepare targets -function _prepare_targets() - local targets_root = target_buildutils.get_root_targets() - target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) -end - -- set compiler function _set_compiler(cmakelists) if config.get("toolchain") then @@ -1341,7 +1334,7 @@ end function make(outputdir) local oldir = os.cd(os.projectdir()) local cmakelists = io.open(path.join(outputdir, "CMakeLists.txt"), "w") - _prepare_targets() + target_cmds.prepare_targets() _generate_cmakelists(cmakelists, outputdir) cmakelists:close() os.cd(oldir) diff --git a/xmake/plugins/project/make/makefile.lua b/xmake/plugins/project/make/makefile.lua index 3d4d5507f..e123943de 100644 --- a/xmake/plugins/project/make/makefile.lua +++ b/xmake/plugins/project/make/makefile.lua @@ -687,10 +687,11 @@ function _add_clean(makefile, outputdir) end function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- open the makefile local makefile = io.open(path.join(outputdir, "makefile"), "w") @@ -715,7 +716,5 @@ function make(outputdir) -- close the makefile makefile:close() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/ninja/build_ninja.lua b/xmake/plugins/project/ninja/build_ninja.lua index 819758447..11be6f4ae 100644 --- a/xmake/plugins/project/ninja/build_ninja.lua +++ b/xmake/plugins/project/ninja/build_ninja.lua @@ -28,6 +28,7 @@ import("core.tool.compiler") import("lib.detect.find_tool") import("lib.detect.find_toolname") import("core.tools.cl.parse_include") +import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) -- this sourcebatch is built? function _sourcebatch_is_built(sourcebatch) @@ -449,10 +450,11 @@ function _add_build_for_targets(ninjafile, outputdir) end function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- open the build.ninja file -- -- we need to change encoding to support msvc_deps_prefix @@ -474,8 +476,6 @@ function make(outputdir) -- close the ninjafile ninjafile:close() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index 332e4858d..debc6fdc5 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -25,6 +25,7 @@ import("core.base.hashset") import("core.project.rule") import("private.utils.batchcmds") import("private.utils.rule_groups") +import("private.action.build.target", {alias = "target_buildutils"}) -- this sourcebatch is built? function _sourcebatch_is_built(sourcebatch) @@ -115,3 +116,9 @@ function get_target_buildcmd_sourcegroups(target, cmds, sourcegroups, opt) end end +-- prepare targets +function prepare_targets() + local targets_root = target_buildutils.get_root_targets() + target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) +end + diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index b6b2e11d7..0a23a1847 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -386,17 +386,14 @@ end -- make vstudio project function make(outputdir, vsinfo) - - -- enter project directory local oldir = os.cd(project.directory()) - -- init solution directory - vsinfo.solution_dir = path.join(outputdir, "vs" .. vsinfo.vstudio_version) + -- prepare targets + target_cmds.prepare_targets() - -- init modes + -- init vsinfo + vsinfo.solution_dir = path.join(outputdir, "vs" .. vsinfo.vstudio_version) vsinfo.modes = _make_vsinfo_modes() - - -- init archs vsinfo.archs = _make_vsinfo_archs() -- load targets @@ -519,7 +516,5 @@ function make(outputdir, vsinfo) -- clear local cache _clear_cache() - - -- leave project directory os.cd(oldir) end -- cgit v1.3.1 From bd6f305cd44634dd9829407d01749ae150a7b2d8 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:45:12 +0800 Subject: improve verilator rules --- xmake/rules/verilator/xmake.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmake/rules/verilator/xmake.lua b/xmake/rules/verilator/xmake.lua index ca60a3289..2042d8233 100644 --- a/xmake/rules/verilator/xmake.lua +++ b/xmake/rules/verilator/xmake.lua @@ -88,9 +88,9 @@ rule("verilator.shared") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) -- cgit v1.3.1 From 117ae687a27f34cf183e967e8688eb3d7e8e4f81 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:45:04 +0800 Subject: compatible with order deps --- xmake/core/project/rule.lua | 12 ++++++++++++ xmake/modules/private/action/build/target.lua | 6 ++++-- xmake/modules/private/utils/rule.lua | 6 ++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index ccab257c7..89ee45f8b 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -61,6 +61,18 @@ function _instance:_build_deps() self._DEPS = self._DEPS or {} self._ORDERDEPS = self._ORDERDEPS or {} instance_deps.load_deps(self, instances, self._DEPS, self._ORDERDEPS, {self:fullname()}) + + -- compatible with `add_deps("foo", {order = true})` + local plaindeps = self:get("deps") + if plaindeps then + for _, depname in ipairs(table.wrap(plaindeps)) do + if self:extraconf("deps", depname, "order") then + self:add("orders", depname, self:name()) + utils.warning("add_deps(%s, {order = true}) has been deprecated, please use `add_orders(%s, %s) instead of it`", + depname, depname, self:name()) + end + end + end end -- clone rule diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index b6d428692..beb6f9cb2 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -223,6 +223,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders local script_group = group_name .. "/" .. instance:fullname() jobgraph:group(script_group, function () if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then @@ -247,7 +248,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) return group_name end @@ -526,6 +527,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) } local has_target_script = false for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders local script_group = group_name .. "/" .. instance:fullname() jobgraph:group(script_group, function () if instance == target then @@ -552,7 +554,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) end -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) return group_name end diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua index 531486f61..ed6aec451 100644 --- a/xmake/modules/private/utils/rule.lua +++ b/xmake/modules/private/utils/rule.lua @@ -46,7 +46,7 @@ end -- end) -- end -- -function build_orders_in_jobgraph(jobgraph, rules, opt) +function build_orders_in_jobgraph(jobgraph, target, rules, opt) opt = opt or {} local root_group = assert(opt.root_group) for _, ruleinst in ipairs(rules) do @@ -55,7 +55,9 @@ function build_orders_in_jobgraph(jobgraph, rules, opt) for _, order in ipairs(orders) do local joborders = {} for _, rulename in ipairs(order) do - local script_group = root_group .. "/" .. rulename + -- we need to use fullname to support namespace + local ruleinst = get_rule(target, rulename) + local script_group = root_group .. "/" .. ruleinst:fullname() if jobgraph:has(script_group) then table.insert(joborders, script_group) end -- cgit v1.3.1 From 537f610d20d3a7f46345a928da4d8f65bad4753e Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:47:01 +0800 Subject: revert 3.0 policy --- xmake/core/project/policy.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index c3754d540..03053cb3e 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -164,7 +164,7 @@ function policy.policies() -- private: it will disable fetch remote package repositories ["network.mode"] = {description = "Set the network mode", type = "string"}, -- Set the compatibility version, e.g. 2.0, 3.0 - ["compatibility.version"] = {description = "Set the compatibility version", type = "string", default = "3.0", values = {"2.0", "3.0"}} + ["compatibility.version"] = {description = "Set the compatibility version", type = "string", values = {"2.0", "3.0"}} } policy._POLICIES = policies end -- cgit v1.3.1 From 7b32b30c9585fafb25f3b024e295ec99a77a2e9b Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:52:15 +0800 Subject: format code --- xmake/rules/platform/windows/idl/xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/platform/windows/idl/xmake.lua b/xmake/rules/platform/windows/idl/xmake.lua index 0381658ec..484bfc150 100644 --- a/xmake/rules/platform/windows/idl/xmake.lua +++ b/xmake/rules/platform/windows/idl/xmake.lua @@ -50,7 +50,7 @@ rule("platform.windows.idl") batchcmds:show_progress(opt.progress, "${color.build.object}compiling.idl %s", sourcefile) batchcmds:vrunv(midl.program, flags, {envs = msvc:runenvs()}) - + local iid_file = path.join(autogendir, name .. "_i.c") local objectfile = target:objectfile(iid_file) table.insert(target:objectfiles(), objectfile) -- cgit v1.3.1 From 9d8707ae3fe665f5f33991f1088d707f334546e5 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:53:02 +0800 Subject: fix idl rule --- xmake/rules/platform/windows/idl/xmake.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xmake/rules/platform/windows/idl/xmake.lua b/xmake/rules/platform/windows/idl/xmake.lua index 484bfc150..f3a907f09 100644 --- a/xmake/rules/platform/windows/idl/xmake.lua +++ b/xmake/rules/platform/windows/idl/xmake.lua @@ -22,10 +22,13 @@ rule("platform.windows.idl") set_extensions(".idl") - on_config(function (target) - local autogendir = path.join(target:autogendir(), "platform/windows/idl") - os.mkdir(autogendir) - target:add("includedirs", autogendir, {public = true}) + on_config("windows", "mingw", function (target) + local sourcebatch = target:sourcebatches()["platform.windows.idl"] + if sourcebatch then + local autogendir = path.join(target:autogendir(), "platform/windows/idl") + os.mkdir(autogendir) + target:add("includedirs", autogendir, {public = true}) + end end) before_buildcmd_file(function (target, batchcmds, sourcefile, opt) -- cgit v1.3.1 From 40f2da18943f7f5e18f838365aebe633c32efc30 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:57:47 +0800 Subject: improve vs runtime --- xmake/core/project/policy.lua | 2 +- xmake/rules/c++/xmake.lua | 20 -------------------- xmake/rules/utils/compiler_runtime/xmake.lua | 11 +++++++++++ 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 03053cb3e..c3754d540 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -164,7 +164,7 @@ function policy.policies() -- private: it will disable fetch remote package repositories ["network.mode"] = {description = "Set the network mode", type = "string"}, -- Set the compatibility version, e.g. 2.0, 3.0 - ["compatibility.version"] = {description = "Set the compatibility version", type = "string", values = {"2.0", "3.0"}} + ["compatibility.version"] = {description = "Set the compatibility version", type = "string", default = "3.0", values = {"2.0", "3.0"}} } policy._POLICIES = policies end diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua index 94bba431e..1b8898226 100644 --- a/xmake/rules/c++/xmake.lua +++ b/xmake/rules/c++/xmake.lua @@ -23,16 +23,6 @@ rule("c.build") add_deps("c.build.pcheader", "c.build.optimization", "c.build.sanitizer") on_build_files("private.action.build.object", {batch = true, distcc = true}) on_config(function (target) - -- enable vs runtime as MD by default - if target:is_plat("windows") and not target:get("runtimes") then - local vs_runtime_default = target:policy("build.c++.msvc.runtime") - if vs_runtime_default and target:has_tool("cc", "cl", "clang_cl") then - if is_mode("debug") then - vs_runtime_default = vs_runtime_default .. "d" - end - target:set("runtimes", vs_runtime_default) - end - end -- https://github.com/xmake-io/xmake/issues/4621 if target:is_plat("windows") and target:is_static() and target:has_tool("cc", "tcc") then target:set("extension", ".a") @@ -49,16 +39,6 @@ rule("c++.build") if target:is_plat("windows") and not target:get("exceptions") then target:set("exceptions", "cxx") end - -- enable vs runtime as MD by default - if target:is_plat("windows") and not target:get("runtimes") then - local vs_runtime_default = target:policy("build.c++.msvc.runtime") - if vs_runtime_default and target:has_tool("cxx", "cl", "clang_cl") then - if is_mode("debug") then - vs_runtime_default = vs_runtime_default .. "d" - end - target:set("runtimes", vs_runtime_default) - end - end -- https://github.com/xmake-io/xmake/issues/4621 if target:is_plat("windows") and target:is_static() and target:has_tool("cxx", "tcc") then target:set("extension", ".a") diff --git a/xmake/rules/utils/compiler_runtime/xmake.lua b/xmake/rules/utils/compiler_runtime/xmake.lua index 1a50fddef..cd8d5297c 100644 --- a/xmake/rules/utils/compiler_runtime/xmake.lua +++ b/xmake/rules/utils/compiler_runtime/xmake.lua @@ -40,5 +40,16 @@ rule("utils.compiler.runtime") end target:set("runtimes", runtimes) end + + -- enable vs runtime as MD by default + if target:is_plat("windows") and not target:get("runtimes") then + local vs_runtime_default = target:policy("build.c++.msvc.runtime") + if vs_runtime_default and target:has_tool("cxx", "cl", "clang_cl") then + if is_mode("debug") then + vs_runtime_default = vs_runtime_default .. "d" + end + target:set("runtimes", vs_runtime_default) + end + end end) -- cgit v1.3.1 From 7b4ba90c3d9b92d2011f78d5373f5850de2dcccf Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:01:55 +0800 Subject: fix target_utils --- xmake/actions/build/builtin/build_binary.lua | 38 ------------- xmake/actions/build/builtin/build_moduleonly.lua | 26 --------- xmake/actions/build/builtin/build_object.lua | 26 --------- xmake/actions/build/builtin/build_shared.lua | 26 --------- xmake/actions/build/builtin/build_static.lua | 26 --------- xmake/actions/build/builtin/link_objects.lua | 68 ------------------------ xmake/actions/build/builtin/prepare_files.lua | 27 ---------- xmake/actions/build/deprecated/kinds/binary.lua | 2 +- xmake/actions/build/deprecated/kinds/shared.lua | 2 +- xmake/actions/build/deprecated/kinds/static.lua | 2 +- 10 files changed, 3 insertions(+), 240 deletions(-) delete mode 100644 xmake/actions/build/builtin/build_binary.lua delete mode 100644 xmake/actions/build/builtin/build_moduleonly.lua delete mode 100644 xmake/actions/build/builtin/build_object.lua delete mode 100644 xmake/actions/build/builtin/build_shared.lua delete mode 100644 xmake/actions/build/builtin/build_static.lua delete mode 100644 xmake/actions/build/builtin/link_objects.lua delete mode 100644 xmake/actions/build/builtin/prepare_files.lua diff --git a/xmake/actions/build/builtin/build_binary.lua b/xmake/actions/build/builtin/build_binary.lua deleted file mode 100644 index 3eb376757..000000000 --- a/xmake/actions/build/builtin/build_binary.lua +++ /dev/null @@ -1,38 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file build_binary.lua --- - --- imports -import("build_object") -import(".target_utils") - -function main(jobgraph, target) - local objects_group = target:fullname() .. "/objects" - local jobsize = jobgraph:size() - jobgraph:group(objects_group, function () - build_object(jobgraph, target) - end) - if jobgraph:size() > jobsize then - local link_group = target:fullname() .. "/link" - jobgraph:group(link_group, function () - target_utils.add_linkjobs(jobgraph, target) - end) - jobgraph:add_orders(objects_group, link_group) - end -end diff --git a/xmake/actions/build/builtin/build_moduleonly.lua b/xmake/actions/build/builtin/build_moduleonly.lua deleted file mode 100644 index e3cc87c27..000000000 --- a/xmake/actions/build/builtin/build_moduleonly.lua +++ /dev/null @@ -1,26 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file build_moduleonly.lua --- - --- imports -import("build_object") - -function main(jobgraph, target) - build_object(jobgraph, target) -end diff --git a/xmake/actions/build/builtin/build_object.lua b/xmake/actions/build/builtin/build_object.lua deleted file mode 100644 index 82268c1e7..000000000 --- a/xmake/actions/build/builtin/build_object.lua +++ /dev/null @@ -1,26 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file build_object.lua --- - --- imports -import(".target_utils") - -function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) -end diff --git a/xmake/actions/build/builtin/build_shared.lua b/xmake/actions/build/builtin/build_shared.lua deleted file mode 100644 index 077489710..000000000 --- a/xmake/actions/build/builtin/build_shared.lua +++ /dev/null @@ -1,26 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file build_shared.lua --- - --- imports -import("build_binary") - -function main(jobgraph, target) - build_binary(jobgraph, target) -end diff --git a/xmake/actions/build/builtin/build_static.lua b/xmake/actions/build/builtin/build_static.lua deleted file mode 100644 index 8b9b2ffaf..000000000 --- a/xmake/actions/build/builtin/build_static.lua +++ /dev/null @@ -1,26 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file build_static.lua --- - --- imports -import("build_binary") - -function main(jobgraph, target) - build_binary(jobgraph, target) -end diff --git a/xmake/actions/build/builtin/link_objects.lua b/xmake/actions/build/builtin/link_objects.lua deleted file mode 100644 index 13a38d0cf..000000000 --- a/xmake/actions/build/builtin/link_objects.lua +++ /dev/null @@ -1,68 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file link_objects.lua --- - --- imports -import("core.base.option") -import("core.tool.linker") -import("core.tool.compiler") -import("core.project.depend") -import("utils.progress") -import("build_object") -import(".target_utils") - --- do link target -function _do_link_target(target, opt) - local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) - local linkflags = linkinst:linkflags({target = target}) - - -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) - local dryrun = option.get("dry-run") - local depvalues = {linkinst:program(), linkflags} - depend.on_changed(function () - local filename = target:filename() - if target:namespace() then - filename = target:namespace() .. "::" .. filename - end - progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) - - local targetfile = target:targetfile() - local objectfiles = target:objectfiles() - local verbose = option.get("verbose") - if verbose then - -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows - print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) - end - - if not dryrun then - assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) - end - end, {dependfile = target:dependfile(), - lastmtime = os.mtime(target:targetfile()), - changed = target:is_rebuilt() or option.get("linkonly"), - values = depvalues, files = depfiles, dryrun = dryrun}) -end - -function main(jobgraph, target) - local linkjob = target:fullname() .. "/link_objects" - jobgraph:add(linkjob, function (index, total, opt) - _do_link_target(target, opt) - end) -end diff --git a/xmake/actions/build/builtin/prepare_files.lua b/xmake/actions/build/builtin/prepare_files.lua deleted file mode 100644 index 320d8cebb..000000000 --- a/xmake/actions/build/builtin/prepare_files.lua +++ /dev/null @@ -1,27 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file prepare_files.lua --- - --- imports -import("core.base.option") -import(".target_utils") - -function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) -end diff --git a/xmake/actions/build/deprecated/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua index 73962becc..727e56857 100644 --- a/xmake/actions/build/deprecated/kinds/binary.lua +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..target_utils") +import("private.utils.target", {alias = "target_utils"}) -- do link target function _do_link_target(target, opt) diff --git a/xmake/actions/build/deprecated/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua index 9b91d67e3..1c24f6009 100644 --- a/xmake/actions/build/deprecated/kinds/shared.lua +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..target_utils") +import("private.utils.target", {alias = "target_utils"}) -- do link target function _do_link_target(target, opt) diff --git a/xmake/actions/build/deprecated/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua index 7057395f9..e6cc0fb44 100644 --- a/xmake/actions/build/deprecated/kinds/static.lua +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("..target_utils") +import("private.utils.target", {alias = "target_utils"}) -- do link target function _do_link_target(target, opt) -- cgit v1.3.1 From 811f8772bbc14d90172aa294b20aed44d41902d9 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:05:18 +0800 Subject: rename to target_buildutils --- xmake/actions/build/deprecated/kinds/binary.lua | 4 ++-- xmake/actions/build/deprecated/kinds/shared.lua | 4 ++-- xmake/actions/build/deprecated/kinds/static.lua | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xmake/actions/build/deprecated/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua index 727e56857..34ef272a9 100644 --- a/xmake/actions/build/deprecated/kinds/binary.lua +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("private.utils.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/deprecated/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua index 1c24f6009..921a8e502 100644 --- a/xmake/actions/build/deprecated/kinds/shared.lua +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("private.utils.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/deprecated/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua index e6cc0fb44..cd7e07ae1 100644 --- a/xmake/actions/build/deprecated/kinds/static.lua +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("private.utils.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () -- cgit v1.3.1 From 77d90d253447ea6992f4f0a81d8df219e23b499c Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:42:30 +0800 Subject: improve target_cmds --- .../modules/private/action/build/build_binary.lua | 6 +- .../private/action/build/build_moduleonly.lua | 4 +- .../modules/private/action/build/build_object.lua | 4 +- .../modules/private/action/build/build_shared.lua | 4 +- .../modules/private/action/build/build_static.lua | 4 +- .../modules/private/action/build/link_objects.lua | 8 +- .../modules/private/action/build/prepare_files.lua | 4 +- xmake/modules/private/action/build/target.lua | 118 +++++++++++++++------ xmake/plugins/project/cmake/cmakelists.lua | 13 +-- xmake/plugins/project/utils/target_cmds.lua | 47 ++++++++ 10 files changed, 156 insertions(+), 56 deletions(-) diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua index 72263e6c8..3863b8550 100644 --- a/xmake/modules/private/action/build/build_binary.lua +++ b/xmake/modules/private/action/build/build_binary.lua @@ -22,16 +22,16 @@ import("build_object") import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) +function main(jobgraph, target, opt) local objects_group = target:fullname() .. "/objects" local jobsize = jobgraph:size() jobgraph:group(objects_group, function () - build_object(jobgraph, target) + build_object(jobgraph, target, opt) end) if jobgraph:size() > jobsize then local link_group = target:fullname() .. "/link" jobgraph:group(link_group, function () - target_buildutils.add_linkjobs(jobgraph, target) + target_buildutils.add_linkjobs(jobgraph, target, opt) end) jobgraph:add_orders(objects_group, link_group) end diff --git a/xmake/modules/private/action/build/build_moduleonly.lua b/xmake/modules/private/action/build/build_moduleonly.lua index e3cc87c27..32c07375e 100644 --- a/xmake/modules/private/action/build/build_moduleonly.lua +++ b/xmake/modules/private/action/build/build_moduleonly.lua @@ -21,6 +21,6 @@ -- imports import("build_object") -function main(jobgraph, target) - build_object(jobgraph, target) +function main(jobgraph, target, opt) + build_object(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua index 2a6baa59a..08f82092b 100644 --- a/xmake/modules/private/action/build/build_object.lua +++ b/xmake/modules/private/action/build/build_object.lua @@ -21,6 +21,6 @@ -- imports import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) - target_buildutils.add_filejobs(jobgraph, target, {job_kind = "build"}) +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_shared.lua b/xmake/modules/private/action/build/build_shared.lua index 077489710..9223093b4 100644 --- a/xmake/modules/private/action/build/build_shared.lua +++ b/xmake/modules/private/action/build/build_shared.lua @@ -21,6 +21,6 @@ -- imports import("build_binary") -function main(jobgraph, target) - build_binary(jobgraph, target) +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_static.lua b/xmake/modules/private/action/build/build_static.lua index 8b9b2ffaf..6d47394ed 100644 --- a/xmake/modules/private/action/build/build_static.lua +++ b/xmake/modules/private/action/build/build_static.lua @@ -21,6 +21,6 @@ -- imports import("build_binary") -function main(jobgraph, target) - build_binary(jobgraph, target) +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua index 25214d27e..396896d00 100644 --- a/xmake/modules/private/action/build/link_objects.lua +++ b/xmake/modules/private/action/build/link_objects.lua @@ -60,9 +60,13 @@ function _do_link_target(target, opt) values = depvalues, files = depfiles, dryrun = dryrun}) end -function main(jobgraph, target) +function main(jobgraph, target, opt) + opt = opt or {} + local buildcmds = opt.buildcmds local linkjob = target:fullname() .. "/link_objects" jobgraph:add(linkjob, function (index, total, opt) - _do_link_target(target, opt) + if not buildcmds then + _do_link_target(target, opt) + end end) end diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua index d8ca351ca..9f0f47ef0 100644 --- a/xmake/modules/private/action/build/prepare_files.lua +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -22,6 +22,6 @@ import("core.base.option") import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) - target_buildutils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index beb6f9cb2..68f6bda53 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -119,14 +119,16 @@ function _add_targetjobs_orders(jobgraph, target, dep, opt) end -- add target jobs for the builtin script -function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) +function add_targetjobs_for_builtin_script(jobgraph, target, opt) + opt = opt or {} + local job_kind = opt.job_kind if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then if job_kind == "prepare" then - import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target) + import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target, opt) elseif job_kind == "link" then - import("private.action.build.link_objects", {anonymous = true})(jobgraph, target) + import("private.action.build.link_objects", {anonymous = true})(jobgraph, target, opt) else - import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target) + import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target, opt) end end end @@ -135,6 +137,7 @@ end function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false + local buildcmds = opt.buildcmds local job_prefix = target:fullname() if target == instance then job_prefix = job_prefix .. "/target" @@ -143,7 +146,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) end -- call script - if not has_script then + if not has_script and not buildcmds then local script_name = opt.script_name local script = instance:script(script_name) if script then @@ -185,9 +188,14 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) if scriptcmd then local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + if buildcmds then + -- only generate cmds and do not run them, use cases: e.g. project generator + scriptcmd(target, buildcmds, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end end) has_script = true end @@ -220,7 +228,8 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) local has_script = false local script_opt = { script_name = script_name, - scriptcmd_name = scriptcmd_name + scriptcmd_name = scriptcmd_name, + buildcmds = opt.buildcmds } for _, instance in ipairs(instances) do -- we need to use this group to sort rule scripts with add_orders @@ -238,7 +247,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- call builtin script, e.g. on_prepare, on_build, ... if not has_script and stage == "" then - add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + add_targetjobs_for_builtin_script(jobgraph, target, opt) end end) @@ -265,10 +274,15 @@ function add_targetjobs(jobgraph, target, opt) _g.pkgenvs = pkgenvs end + local buildcmds = opt.buildcmds local job_kind = opt.job_kind local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) local job_end = string.format("%s/end_%s", target:fullname(), job_kind) jobgraph:add(job_begin, function (index, total, opt) + if buildcmds then + return + end + -- enter package environments -- https://github.com/xmake-io/xmake/issues/4033 -- @@ -293,6 +307,10 @@ function add_targetjobs(jobgraph, target, opt) end) jobgraph:add(job_end, function (index, total, opt) + if buildcmds then + return + end + -- restore environments if target:pkgenvs() then pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() @@ -307,9 +325,17 @@ function add_targetjobs(jobgraph, target, opt) end) -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end @@ -341,6 +367,7 @@ end function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) opt = opt or {} local has_script = false + local buildcmds = opt.buildcmds local job_prefix = target:fullname() local file_group = sourcebatch.rulename if target == instance then @@ -350,7 +377,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) end -- call script files - if not has_script then + if not has_script and not buildcmds then local script_files_name = opt.script_files_name local script_files = instance:script(script_files_name) if script_files then @@ -383,7 +410,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) end -- call script file - if not has_script then + if not has_script and not buildcmds then local script_file_name = opt.script_file_name local script_file = instance:script(script_file_name) if script_file then @@ -434,9 +461,14 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local distcc = instance:extraconf(scriptcmd_files_name, "distcc") local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + scriptcmd_files(target, buildcmds, sourcebatch, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end end) has_script = true end @@ -455,12 +487,20 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local distcc = instance:extraconf(scriptcmd_file_name, "distcc") local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, buildcmds, sourcefile, {sourcekind = sourcekind}) + end + else + local batchcmds_ = batchcmds.new({target = target}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end) has_script = true end @@ -473,6 +513,7 @@ end -- function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} + local buildcmds = opt.buildcmds local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" @@ -523,7 +564,8 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) script_file_name = script_file_name, script_files_name = script_files_name, scriptcmd_file_name = scriptcmd_file_name, - scriptcmd_files_name = scriptcmd_files_name + scriptcmd_files_name = scriptcmd_files_name, + buildcmds = buildcmds } local has_target_script = false for _, instance in ipairs(instances) do @@ -575,9 +617,17 @@ function add_filejobs(jobgraph, target, opt) end -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) - local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) - local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + end jobgraph:add_orders(group_before, group, group_after) end @@ -608,9 +658,17 @@ end function add_linkjobs(jobgraph, target, opt) opt = opt or {} opt.job_kind = "link" - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end jobgraph:add_orders(group_before, group, group_after) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index f826cc453..9d2fc453a 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -29,7 +29,6 @@ import("core.project.rule") import("core.platform.platform") import("lib.detect.find_tool") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -1201,9 +1200,6 @@ end -- add target custom commands function _add_target_custom_commands(cmakelists, target, outputdir) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules if _can_native_support_for_cxxmodules() then @@ -1212,17 +1208,12 @@ function _add_target_custom_commands(cmakelists, target, outputdir) -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index debc6fdc5..c25a2c3e3 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -122,3 +122,50 @@ function prepare_targets() target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) end +-- get target buildcmds +function get_target_buildcmds(target, opt) + opt = opt or {} + local progress_wrapper = {} + progress_wrapper.current = function () + return count + end + progress_wrapper.total = function () + return total + end + progress_wrapper.percent = function () + if total and total > 0 then + return math.floor((count * 100) / total) + else + return 0 + end + end + debug.setmetatable(progress_wrapper, { + __tostring = function () + -- we do not output any progress info for the project generators + return "" + end + }) + local buildcmds = batchcmds.new({target = target}) + local jobgraph = target_buildutils.get_targetjobs({target}, { + job_kind = "build", + buildcmds = buildcmds, + with_stages = hashset.from(opt.stages or {}), + ignored_rules = hashset.from(opt.ignored_rules or {}), + progress = progress_wrapper}) + if jobgraph and not jobgraph:empty() then + local jobqueue = jobgraph:build() + while true do + local job = jobqueue:getfree() + if job then + if job.run then + job.run() + end + jobqueue:remove(job) + else + break + end + end + end + return buildcmds:cmds() +end + -- cgit v1.3.1 From 4109abcddf430a8fad253566b23db6f8c73b5366 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:44:19 +0800 Subject: ignore rules and stages --- xmake/modules/private/action/build/target.lua | 38 ++++++++++++++++----------- xmake/plugins/project/utils/target_cmds.lua | 8 +++--- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index 68f6bda53..477dd3023 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -208,6 +208,7 @@ end function add_targetjobs_with_stage(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind + local ignored_rules = opt.ignored_rules -- the group name, e.g. foo/after_prepare, bar/before_build local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) @@ -220,8 +221,11 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- call target and rules script local instances = {target} - for _, r in ipairs(target:orderules()) do - table.insert(instances, r) + for _, ruleinst in ipairs(target:orderules()) do + -- we only ignore some builtin rules, so we need not to use fullname. + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + table.insert(instances, ruleinst) + end end local jobsize = jobgraph:size() jobgraph:group(group_name, function () @@ -491,7 +495,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) if buildcmds then local sourcekind = sourcebatch.sourcekind for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, buildcmds, sourcefile, {sourcekind = sourcekind}) + scriptcmd_file(target, buildcmds, sourcefile, {progress = opt.progress, sourcekind = sourcekind}) end else local batchcmds_ = batchcmds.new({target = target}) @@ -514,6 +518,7 @@ end function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} local buildcmds = opt.buildcmds + local ignored_rules = opt.ignored_rules local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" @@ -538,20 +543,23 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, sourcebatch in pairs(sourcebatches) do local rulename = sourcebatch.rulename if rulename then + -- we only ignore some builtin rules, so we need not to use fullname. local ruleinst = rule_utils.get_rule(target, rulename) - sourcebatches_map[ruleinst] = sourcebatch - -- avoid duplicate scripts being called twice in the target, - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - if ruleinst:script("build_file") or ruleinst:script("build_files") then - table.insert(sourcebatches_for_target, sourcebatch) + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + table.insert(instances, ruleinst) end - table.insert(instances, ruleinst) else table.insert(sourcebatches_for_target, sourcebatch) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index c25a2c3e3..835117d50 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -150,17 +150,19 @@ function get_target_buildcmds(target, opt) job_kind = "build", buildcmds = buildcmds, with_stages = hashset.from(opt.stages or {}), - ignored_rules = hashset.from(opt.ignored_rules or {}), - progress = progress_wrapper}) + ignored_rules = hashset.from(opt.ignored_rules or {})}) if jobgraph and not jobgraph:empty() then + local total = jobgraph:size() + local index = 0 local jobqueue = jobgraph:build() while true do local job = jobqueue:getfree() if job then if job.run then - job.run() + job.run(index, total, {progress = progress_wrapper}) end jobqueue:remove(job) + index = index + 1 else break end -- cgit v1.3.1 From 95e69eeea88e1ff30f67ef94bab7ba1652b026a4 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:46:21 +0800 Subject: fix link jobs --- xmake/modules/private/action/build/target.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index 477dd3023..c302408ac 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -664,7 +664,7 @@ end -- add link jobs for the given target function add_linkjobs(jobgraph, target, opt) - opt = opt or {} + opt = table.clone(opt or {}) opt.job_kind = "link" local with_stages = opt.with_stages local group, group_before, group_after -- cgit v1.3.1 From 260f07164381c235422ec7150e26396604fabdb5 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:47:05 +0800 Subject: improve makefile generator --- xmake/plugins/project/make/makefile.lua | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/xmake/plugins/project/make/makefile.lua b/xmake/plugins/project/make/makefile.lua index e123943de..79277b3f1 100644 --- a/xmake/plugins/project/make/makefile.lua +++ b/xmake/plugins/project/make/makefile.lua @@ -27,7 +27,6 @@ import("core.language.language") import("core.platform.platform") import("lib.detect.find_tool") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) -- tranlate path @@ -469,14 +468,11 @@ function _add_build_phony(makefile, target) end -- add custom commands before building target -function _add_build_custom_commands_before(makefile, target, sourcegroups, outputdir) +function _add_build_custom_commands_before(makefile, target, outputdir) -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups) + local cmds_before = target_cmds.get_target_buildcmds(target, {stages = {"before", "on"}}) local targetname = target:name() local label = "precmds_" .. targetname @@ -494,10 +490,8 @@ function _add_build_custom_commands_before(makefile, target, sourcegroups, outpu end -- add custom commands after building target -function _add_build_custom_commands_after(makefile, target, sourcegroups, outputdir) - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after"}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after"}) +function _add_build_custom_commands_after(makefile, target, outputdir) + local cmds_after = target_cmds.get_target_buildcmds(target, {stages = {"after"}}) if #cmds_after > 0 then for _, cmd in ipairs(cmds_after) do local command = _get_command_string(cmd, outputdir) @@ -514,11 +508,8 @@ function _add_build_target(makefile, target, targetflags, outputdir) -- https://github.com/xmake-io/xmake/issues/2337 target:data_set("plugin.project.kind", "makefile") - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- add custom commands before building target - local precmds_label = _add_build_custom_commands_before(makefile, target, sourcegroups, outputdir) + local precmds_label = _add_build_custom_commands_before(makefile, target, outputdir) -- is phony target? if target:is_phony() then @@ -602,7 +593,7 @@ function _add_build_target(makefile, target, targetflags, outputdir) makefile:writef("\t$(VV)%s\n", command) -- add custom commands after building target - _add_build_custom_commands_after(makefile, target, sourcegroups, outputdir) + _add_build_custom_commands_after(makefile, target, outputdir) -- end makefile:print("") -- cgit v1.3.1 From 699e40f3e45469b3597f5fa24ad6bb2c922b8a4b Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:48:29 +0800 Subject: move rule_group to deprecated --- xmake/actions/build/deprecated/kinds/object.lua | 2 +- .../actions/build/deprecated/kinds/rule_groups.lua | 95 ++++++++++++++++++++++ xmake/modules/private/utils/rule_groups.lua | 95 ---------------------- xmake/plugins/project/clang/compile_commands.lua | 14 +--- xmake/plugins/project/utils/target_cmds.lua | 90 -------------------- xmake/plugins/project/vstudio/impl/vs201x.lua | 17 ++-- 6 files changed, 103 insertions(+), 210 deletions(-) create mode 100644 xmake/actions/build/deprecated/kinds/rule_groups.lua delete mode 100644 xmake/modules/private/utils/rule_groups.lua diff --git a/xmake/actions/build/deprecated/kinds/object.lua b/xmake/actions/build/deprecated/kinds/object.lua index b5359f79e..c4ed8721a 100644 --- a/xmake/actions/build/deprecated/kinds/object.lua +++ b/xmake/actions/build/deprecated/kinds/object.lua @@ -25,7 +25,7 @@ import("core.project.config") import("core.project.project") import("async.runjobs") import("private.utils.batchcmds") -import("private.utils.rule_groups") +import("rule_groups") -- has scripts for the custom rule function _has_scripts_for_rule(ruleinst, suffix) diff --git a/xmake/actions/build/deprecated/kinds/rule_groups.lua b/xmake/actions/build/deprecated/kinds/rule_groups.lua new file mode 100644 index 000000000..d64a75d62 --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/rule_groups.lua @@ -0,0 +1,95 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file rule_groups.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") + +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + +-- get max depth of rule +function _get_rule_max_depth(target, ruleinst, depth) + local max_depth = depth + for _, depname in ipairs(ruleinst:get("deps")) do + local dep = get_rule(target, depname) + local dep_depth = depth + if ruleinst:extraconf("deps", depname, "order") then + dep_depth = dep_depth + 1 + end + local cur_depth = _get_rule_max_depth(target, dep, dep_depth) + if cur_depth > max_depth then + max_depth = cur_depth + end + end + return max_depth +end + +-- build sourcebatch groups for target +function _build_sourcebatch_groups_for_target(groups, target, sourcebatches) + local group = groups[1] + for _, sourcebatch in pairs(sourcebatches) do + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local item = group[rulename] or {} + item.target = target + item.sourcebatch = sourcebatch + group[rulename] = item + end +end + +-- build sourcebatch groups for rules +function _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) + for _, sourcebatch in pairs(sourcebatches) do + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local ruleinst = get_rule(target, rulename) + local depth = _get_rule_max_depth(target, ruleinst, 1) + local group = groups[depth] + if group == nil then + group = {} + groups[depth] = group + end + local item = group[rulename] or {} + item.rule = ruleinst + item.sourcebatch = sourcebatch + group[rulename] = item + end +end + +-- build sourcebatch groups by rule dependencies order, e.g. `add_deps("qt.ui", {order = true})` +-- +-- @see https://github.com/xmake-io/xmake/issues/2814 +-- +function build_sourcebatch_groups(target, sourcebatches) + local groups = {{}} + _build_sourcebatch_groups_for_target(groups, target, sourcebatches) + _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) + if #groups > 0 then + groups = table.reverse(groups) + end + return groups +end + diff --git a/xmake/modules/private/utils/rule_groups.lua b/xmake/modules/private/utils/rule_groups.lua deleted file mode 100644 index d64a75d62..000000000 --- a/xmake/modules/private/utils/rule_groups.lua +++ /dev/null @@ -1,95 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file rule_groups.lua --- - --- imports -import("core.base.option") -import("core.project.rule") -import("core.project.config") -import("core.project.project") - --- get rule --- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target -function get_rule(target, rulename) - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - return ruleinst -end - --- get max depth of rule -function _get_rule_max_depth(target, ruleinst, depth) - local max_depth = depth - for _, depname in ipairs(ruleinst:get("deps")) do - local dep = get_rule(target, depname) - local dep_depth = depth - if ruleinst:extraconf("deps", depname, "order") then - dep_depth = dep_depth + 1 - end - local cur_depth = _get_rule_max_depth(target, dep, dep_depth) - if cur_depth > max_depth then - max_depth = cur_depth - end - end - return max_depth -end - --- build sourcebatch groups for target -function _build_sourcebatch_groups_for_target(groups, target, sourcebatches) - local group = groups[1] - for _, sourcebatch in pairs(sourcebatches) do - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local item = group[rulename] or {} - item.target = target - item.sourcebatch = sourcebatch - group[rulename] = item - end -end - --- build sourcebatch groups for rules -function _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) - for _, sourcebatch in pairs(sourcebatches) do - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = get_rule(target, rulename) - local depth = _get_rule_max_depth(target, ruleinst, 1) - local group = groups[depth] - if group == nil then - group = {} - groups[depth] = group - end - local item = group[rulename] or {} - item.rule = ruleinst - item.sourcebatch = sourcebatch - group[rulename] = item - end -end - --- build sourcebatch groups by rule dependencies order, e.g. `add_deps("qt.ui", {order = true})` --- --- @see https://github.com/xmake-io/xmake/issues/2814 --- -function build_sourcebatch_groups(target, sourcebatches) - local groups = {{}} - _build_sourcebatch_groups_for_target(groups, target, sourcebatches) - _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) - if #groups > 0 then - groups = table.reverse(groups) - end - return groups -end - diff --git a/xmake/plugins/project/clang/compile_commands.lua b/xmake/plugins/project/clang/compile_commands.lua index c36cb8ade..4cf6eb1ab 100644 --- a/xmake/plugins/project/clang/compile_commands.lua +++ b/xmake/plugins/project/clang/compile_commands.lua @@ -27,7 +27,6 @@ import("core.project.project") import("core.language.language") import("private.utils.batchcmds") import("private.utils.executable_path") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("actions.test.main", {rootdir = os.programdir(), alias = "test_action"}) @@ -242,25 +241,16 @@ end -- add target commands function _add_target_commands(jsonfile, target) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before"}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups) + local cmds_before = target_cmds.get_target_buildcmds(target, {stages = {"before", "on"}}) _add_target_custom_commands(jsonfile, target, "before", cmds_before) -- add target source commands _add_target_source_commands(jsonfile, target) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after"}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after"}) + local cmds_after = target_cmds.get_target_buildcmds(target, {stages = {"after"}}) _add_target_custom_commands(jsonfile, target, "after", cmds_after) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index 835117d50..b18acb943 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -24,98 +24,8 @@ import("core.project.config") import("core.base.hashset") import("core.project.rule") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("private.action.build.target", {alias = "target_buildutils"}) --- this sourcebatch is built? -function _sourcebatch_is_built(sourcebatch) - -- we can only use rulename to filter them because sourcekind may be bound to multiple rules - local rulename = sourcebatch.rulename - if rulename == "c.build" or rulename == "c++.build" - or rulename == "asm.build" or rulename == "cuda.build" - or rulename == "objc.build" or rulename == "objc++.build" - or rulename == "win.sdk.resource" then - return true - end -end - --- get target buildcmd commands -function get_target_buildcmd(target, cmds, opt) - opt = opt or {} - local suffix = opt.suffix - local ignored_rules = hashset.from(opt.ignored_rules or {}) - for _, ruleinst in ipairs(target:orderules()) do - if not ignored_rules:has(ruleinst:name()) then - local scriptname = "buildcmd" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - end - end -end - --- get target buildcmd_files commands -function get_target_buildcmd_files(target, cmds, sourcebatch, opt) - opt = opt or {} - - -- get rule - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - local ignored_rules = hashset.from(opt.ignored_rules or {}) - if ignored_rules:has(ruleinst:name()) then - return - end - - -- generate commands for xx_buildcmd_files - local suffix = opt.suffix - local scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcebatch, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - - -- generate commands for xx_buildcmd_file - if not script then - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcefile, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - end - end -end - --- get target buildcmd commands of source group -function get_target_buildcmd_sourcegroups(target, cmds, sourcegroups, opt) - for idx, group in irpairs(sourcegroups) do - for _, item in pairs(group) do - -- buildcmd scripts are always in rule, so we need to ignore target item (item.target). - local sourcebatch = item.sourcebatch - if item.rule then - if not _sourcebatch_is_built(sourcebatch) then - get_target_buildcmd_files(target, cmds, sourcebatch, opt) - end - end - end - end -end - -- prepare targets function prepare_targets() local targets_root = target_buildutils.get_root_targets() diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index 0a23a1847..e3b9ddc95 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -39,7 +39,6 @@ import("private.action.require.install", {alias = "install_requires"}) import("private.action.run.runenvs") import("actions.config.configfiles", {alias = "generate_configfiles", rootdir = os.programdir()}) import("private.utils.batchcmds") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) function _translate_path(dir, vcxprojdir) @@ -132,24 +131,18 @@ function _make_custom_commands(target, vcxprojdir) return _translate_path(p, vcxprojdir) end) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules = _get_cxxmodules_rules() -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + -- rule.on_buildcmd_files should also be executed before building the target + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) local commands = {} for _, cmd in ipairs(cmds_before) do -- cgit v1.3.1 From d9a5ab70746010c846b815c2b5e14447d268b96f Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:24:06 +0800 Subject: update policy --- xmake/core/project/policy.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index c3754d540..a476d5bdb 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -97,7 +97,7 @@ function policy.policies() -- Enable windows UAC and set level, e.g. invoker, admin, highest ["windows.manifest.uac"] = {description = "Enable windows manifest UAC.", type = "string"}, -- Enable ui access for windows UAC - ["windows.manifest.uac.ui"] = {description = "Enable windows manifest UAC.", type = "boolean"}, + ["windows.manifest.uac.ui"] = {description = "Enable ui access for windows UAC.", type = "boolean"}, -- Automatically build before running ["run.autobuild"] = {description = "Automatically build before running.", type = "boolean"}, -- Enable install rpath -- cgit v1.3.1 From db9a7049fbf5a870a167ee8b2683cc226dcaa55a Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 00:46:43 +0800 Subject: enable build.autorun by default --- xmake/core/project/project.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua index 7b42d47ea..956e3c41d 100644 --- a/xmake/core/project/project.lua +++ b/xmake/core/project/project.lua @@ -885,6 +885,7 @@ function project._init_default_policies() if semver.compare(compatibility_version, "3.0") >= 0 then policy.set_default("package.cmake_generator.ninja", true) policy.set_default("build.c++.msvc.runtime", "MD") + policy.set_default("run.autobuild", true) else policy.set_default("package.cmake_generator.ninja", false) policy.set_default("build.c++.msvc.runtime", "MT") -- cgit v1.3.1 From 93a2a58bf6f1654ded6275e39b9def0577ce1e5a Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 00:52:36 +0800 Subject: fix cxxmodule cache --- xmake/actions/config/main.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/actions/config/main.lua b/xmake/actions/config/main.lua index 0f69142a5..3dfeb7142 100644 --- a/xmake/actions/config/main.lua +++ b/xmake/actions/config/main.lua @@ -358,7 +358,6 @@ force to build in current directory via run `xmake -P .`]], os.projectdir()) localcache.clear("option") localcache.clear("package") localcache.clear("toolchain") - localcache.clear("cxxmodules") localcache.set("config", "recheck", true) localcache.save() -- cgit v1.3.1 From cf75a18fc72cf469a3cfc9aa5ff550deb2b29c27 Mon Sep 17 00:00:00 2001 From: Doekin Date: Thu, 10 Apr 2025 09:23:48 +0800 Subject: improve `emscripten` detection by including `emcc.py` search --- xmake/modules/detect/sdks/find_emsdk.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/xmake/modules/detect/sdks/find_emsdk.lua b/xmake/modules/detect/sdks/find_emsdk.lua index cbc6998b2..ec8553692 100644 --- a/xmake/modules/detect/sdks/find_emsdk.lua +++ b/xmake/modules/detect/sdks/find_emsdk.lua @@ -33,10 +33,17 @@ function _find_emsdkdir(sdkdir) table.insert(paths, sdkdir) end table.insert(paths, "$(env EMSDK)") + if is_host("linux") then + table.join2(paths, {"/usr/share/emscripten/", "/usr/lib/emscripten/"}) + end local emsdk = find_file("emsdk.py", paths, {suffixes = subdirs}) if emsdk then return path.directory(emsdk) end + local emcc_py = find_file("emcc.py", paths, {suffixes = subdirs}) + if emcc_py then + return path.directory(emcc_py) + end end -- find emsdk @@ -53,6 +60,7 @@ function _find_emsdk(sdkdir) local subdirs = {} table.insert(subdirs, path.join("*", "emscripten")) local emcc = find_file("emcc", sdkdir, {suffixes = subdirs}) + emcc = emcc or find_file("emcc.py", sdkdir) if emcc then emscripten = path.directory(emcc) end -- cgit v1.3.1 From 559bcb7728c99d69a3e61d73410aed73a316fa7f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:31:25 +0800 Subject: update changelog --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 915d92307..a2266582b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## master (unreleased) +### Changes + +* [#6202](https://github.com/xmake-io/xmake/issues/6202): Improve rule API and build dependency order +* [#5624](https://github.com/xmake-io/xmake/discussions/5624): Enable auto build when calling xmake run by default +* [#5526](https://github.com/xmake-io/xmake/discussions/5526): Use MD/MDd runtimes for msvc by default +* [#5545](https://github.com/xmake-io/xmake/discussions/5545): Use ninja generator for cmake package by default + ## v2.9.9 ### New features @@ -1968,6 +1975,13 @@ ## master (开发中) +### 改进 + +* [#6202](https://github.com/xmake-io/xmake/issues/6202): 改进 rule API 和构建顺序支持,提供统一 jobgraph 调度 +* [#5624](https://github.com/xmake-io/xmake/discussions/5624): `xmake run` 运行默认自动构建 +* [#5526](https://github.com/xmake-io/xmake/discussions/5526): msvc 默认切换到 MD/MDd 运行时 +* [#5545](https://github.com/xmake-io/xmake/discussions/5545): 构建 cmake 包,默认使用 Ninja 生成器 + ## v2.9.9 ### 新特性 -- cgit v1.3.1 From 45ca6fea545babb73773b536e6e056e6ca71fe4f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:32:21 +0800 Subject: suppress warnings --- xmake/modules/private/action/require/impl/package.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 87547ab3f..02fac7a3e 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -222,7 +222,7 @@ function _load_require(require_str, requires_extra, opt) -- check require options local extra_options = hashset.of("plat", "arch", "kind", "host", "targetos", "alias", "group", "system", "option", "default", "optional", "debug", - "verify", "external", "private", "build", "configs", "version") + "verify", "external", "private", "build", "configs", "version", "public") for name, value in pairs(require_extra) do if not extra_options:has(name) then wprint("add_requires(\"%s\") has unknown option: {%s=%s}!", require_str, name, tostring(value)) -- cgit v1.3.1 From 55dc13e2761a4b832bb8c7953043c494f50d90b7 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:33:09 +0800 Subject: fix recheck --- xmake/actions/config/main.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/xmake/actions/config/main.lua b/xmake/actions/config/main.lua index 3dfeb7142..899748d2e 100644 --- a/xmake/actions/config/main.lua +++ b/xmake/actions/config/main.lua @@ -95,6 +95,7 @@ function _need_check(changed) if not changed then if os.mtime(path.join(os.programdir(), "core", "main.lua")) > os.mtime(config.filepath()) then changed = true + os.touch(config.filepath(), {mtime = os.time()}) end end return changed -- cgit v1.3.1 From 803bf74e1eb6d589fc2d5709a2c4861a14241c2f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:37:08 +0800 Subject: improve run.autobuild --- xmake/actions/build/main.lua | 13 +++--- xmake/actions/run/main.lua | 2 +- xmake/core/base/option.lua | 2 - .../sandbox/modules/import/core/base/option.lua | 51 +++++----------------- 4 files changed, 20 insertions(+), 48 deletions(-) diff --git a/xmake/actions/build/main.lua b/xmake/actions/build/main.lua index 5a6aa1535..d9ad6818f 100644 --- a/xmake/actions/build/main.lua +++ b/xmake/actions/build/main.lua @@ -160,7 +160,8 @@ function build_targets(targetnames, opt) _do_project_rules("build_after") end -function main() +function main(opt) + opt = opt or {} -- try building it using third-party buildsystem if xmake.lua not exists if not os.isfile(project.rootfile()) and _try_build() then @@ -206,9 +207,11 @@ function main() project.unlock() -- trace - local str = "" - if build_time then - str = string.format(", spent %ss", build_time / 1000) + if not opt.disable_dump then + local str = "" + if build_time then + str = string.format(", spent %ss", build_time / 1000) + end + progress.show(100, "${color.success}build ok%s", str) end - progress.show(100, "${color.success}build ok%s", str) end diff --git a/xmake/actions/run/main.lua b/xmake/actions/run/main.lua index ab2db5cd1..72fe8bbac 100644 --- a/xmake/actions/run/main.lua +++ b/xmake/actions/run/main.lua @@ -196,7 +196,7 @@ function main() -- we need clear the previous config and reload it -- to avoid trigger recheck configs config.clear() - task.run("build", {target = option.get("target"), all = option.get("all")}) + task.run("build", {target = option.get("target"), all = option.get("all")}, {disable_dump = true}) end -- load targets diff --git a/xmake/core/base/option.lua b/xmake/core/base/option.lua index 8448788c5..62551dda3 100644 --- a/xmake/core/base/option.lua +++ b/xmake/core/base/option.lua @@ -51,8 +51,6 @@ function option._translate(menu) -- save menu option._MENU = menu - - -- ok return true end diff --git a/xmake/core/sandbox/modules/import/core/base/option.lua b/xmake/core/sandbox/modules/import/core/base/option.lua index 42ac2477a..852115667 100644 --- a/xmake/core/sandbox/modules/import/core/base/option.lua +++ b/xmake/core/sandbox/modules/import/core/base/option.lua @@ -18,33 +18,23 @@ -- @file option.lua -- --- define module -local sandbox_core_base_option = sandbox_core_base_option or {} - -- load modules local table = require("base/table") local option = require("base/option") local raise = require("sandbox/modules/raise") --- get the option value -function sandbox_core_base_option.get(name) - return option.get(name) -end - --- set the option value -function sandbox_core_base_option.set(name, value) - option.set(name, value) -end +-- define module +local sandbox_core_base_option = sandbox_core_base_option or {} --- get the default option value -function sandbox_core_base_option.default(name) - return option.default(name) -end - --- get the given task menu -function sandbox_core_base_option.taskmenu(taskname) - return option.taskmenu(taskname) -end +-- inherit some builtin interfaces +sandbox_core_base_option.get = option.get +sandbox_core_base_option.set = option.set +sandbox_core_base_option.default = option.default +sandbox_core_base_option.save = option.save +sandbox_core_base_option.restore = option.restore +sandbox_core_base_option.boolean = option.boolean +sandbox_core_base_option.taskname = option.taskname +sandbox_core_base_option.taskmenu = option.taskmenu -- get the options function sandbox_core_base_option.options() @@ -68,8 +58,6 @@ end -- parse arguments with the given options function sandbox_core_base_option.raw_parse(argv, options, opt) - - -- check assert(argv and options) -- parse it @@ -82,8 +70,6 @@ end -- parse arguments with the given options function sandbox_core_base_option.parse(argv, options, ...) - - -- check assert(argv and options) -- add common options @@ -117,20 +103,5 @@ function sandbox_core_base_option.parse(argv, options, ...) return results end --- save context -function sandbox_core_base_option.save(taskname) - return option.save(taskname) -end - --- restore context -function sandbox_core_base_option.restore() - option.restore() -end - --- get the boolean value -function sandbox_core_base_option.boolean(value) - return option.boolean(value) -end - -- return module return sandbox_core_base_option -- cgit v1.3.1 From 389765de963238987ebe27b8237554bdffb7bfdc Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Thu, 10 Apr 2025 13:05:11 +0800 Subject: package(autoconf): resolve issue with backslash-separated `CXX` path in MSYS --- xmake/modules/package/tools/autoconf.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/xmake/modules/package/tools/autoconf.lua b/xmake/modules/package/tools/autoconf.lua index 8d312a76a..821ab759f 100644 --- a/xmake/modules/package/tools/autoconf.lua +++ b/xmake/modules/package/tools/autoconf.lua @@ -434,6 +434,7 @@ function buildenvs(package, opt) if is_host("windows") then envs.CC = _translate_windows_bin_path(envs.CC) + envs.CXX = _translate_windows_bin_path(envs.CXX) envs.AS = _translate_windows_bin_path(envs.AS) envs.AR = _translate_windows_bin_path(envs.AR) envs.LD = _translate_windows_bin_path(envs.LD) -- cgit v1.3.1 From 1268cad62854df59e71757a55fddd74ebde6c65f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:41:42 +0800 Subject: improve cmake:find_package #6296 --- xmake/modules/package/manager/cmake/find_package.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xmake/modules/package/manager/cmake/find_package.lua b/xmake/modules/package/manager/cmake/find_package.lua index 6c3a9f194..4b1c390f0 100644 --- a/xmake/modules/package/manager/cmake/find_package.lua +++ b/xmake/modules/package/manager/cmake/find_package.lua @@ -81,6 +81,8 @@ function _find_package(cmake, name, opt) if moduledirs then for _, moduledir in ipairs(moduledirs) do cmakefile:print("list(APPEND CMAKE_MODULE_PATH \"%s\")", (moduledir:gsub("\\", "/"))) + -- https://github.com/xmake-io/xmake/issues/6296 + cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (moduledir:gsub("\\", "/"))) end end -- e.g. set(Boost_USE_STATIC_LIB ON) -- cgit v1.3.1 From fa122559e2fcb560f7be7b6f61241b8caf524c65 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:42:03 +0800 Subject: improve cmake:find_package again #6296 --- xmake/modules/package/manager/cmake/find_package.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/xmake/modules/package/manager/cmake/find_package.lua b/xmake/modules/package/manager/cmake/find_package.lua index 4b1c390f0..4c71291d7 100644 --- a/xmake/modules/package/manager/cmake/find_package.lua +++ b/xmake/modules/package/manager/cmake/find_package.lua @@ -81,10 +81,16 @@ function _find_package(cmake, name, opt) if moduledirs then for _, moduledir in ipairs(moduledirs) do cmakefile:print("list(APPEND CMAKE_MODULE_PATH \"%s\")", (moduledir:gsub("\\", "/"))) - -- https://github.com/xmake-io/xmake/issues/6296 - cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (moduledir:gsub("\\", "/"))) end end + -- https://github.com/xmake-io/xmake/issues/6296 + local prefixdirs = configs.prefixdirs or opt.prefixdirs + if prefixdirs then + for _, prefixdir in ipairs(prefixdirs) do + cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (prefixdir:gsub("\\", "/"))) + end + end + -- e.g. set(Boost_USE_STATIC_LIB ON) local presets = configs.presets or opt.presets if presets then -- cgit v1.3.1