From 00ebba39f7e81d2b52bfb3be458f9825e6d7c070 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 14 Mar 2025 00:53:08 +0800 Subject: add jobgraph stub --- xmake/modules/async/jobgraph.lua | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 xmake/modules/async/jobgraph.lua (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua new file mode 100644 index 000000000..519544cf2 --- /dev/null +++ b/xmake/modules/async/jobgraph.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file jobgraph.lua +-- + +-- imports +import("core.base.object") +import("core.base.graph") + +-- define module +local jobgraph = jobgraph or object {_init = {"_size"}} + +-- tostring +function jobgraph:__tostring() + return "" +end + +-- new a jobgraph +function new() + return jobgraph {0} +end -- cgit v1.3.1 From 1621db050a13474f4d79d02c4d306013770d35b4 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 18 Mar 2025 23:55:15 +0800 Subject: add jobgraph.new --- tests/modules/async/jobgraph.lua | 4 ++-- xmake/modules/async/jobgraph.lua | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/jobgraph.lua b/tests/modules/async/jobgraph.lua index dd8cb05f8..7a0b87c65 100644 --- a/tests/modules/async/jobgraph.lua +++ b/tests/modules/async/jobgraph.lua @@ -1,7 +1,7 @@ -import("core.base.scheduler") import("async.jobgraph") function main() - + local gh = jobgraph.new() + print(gh) end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 519544cf2..3fd36194a 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -20,17 +20,23 @@ -- imports import("core.base.object") +import("core.base.list") import("core.base.graph") -- define module -local jobgraph = jobgraph or object {_init = {"_size"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_graph"}} + +-- get jobs +function jobgraph:jobs() + return self._jobs +end -- tostring function jobgraph:__tostring() - return "" + return string.format("", self:jobs():size()) end -- new a jobgraph function new() - return jobgraph {0} + return jobgraph {list.new(), graph.new(true)} end -- cgit v1.3.1 From 6785ce5822f298ce64684548dfc9d35ec8dd5030 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:03:25 +0800 Subject: improve runjobs tests --- tests/modules/async/jobgraph.lua | 7 ------ tests/modules/async/run_callback.lua | 19 ++++++++++++++++ tests/modules/async/run_jobgraph.lua | 28 +++++++++++++++++++++++ tests/modules/async/run_jobpool.lua | 28 +++++++++++++++++++++++ tests/modules/async/runjobs.lua | 43 ------------------------------------ xmake/modules/async/jobgraph.lua | 15 +++++++++++++ 6 files changed, 90 insertions(+), 50 deletions(-) delete mode 100644 tests/modules/async/jobgraph.lua create mode 100644 tests/modules/async/run_callback.lua create mode 100644 tests/modules/async/run_jobgraph.lua create mode 100644 tests/modules/async/run_jobpool.lua delete mode 100644 tests/modules/async/runjobs.lua (limited to 'xmake/modules') diff --git a/tests/modules/async/jobgraph.lua b/tests/modules/async/jobgraph.lua deleted file mode 100644 index 7a0b87c65..000000000 --- a/tests/modules/async/jobgraph.lua +++ /dev/null @@ -1,7 +0,0 @@ -import("async.jobgraph") - -function main() - local gh = jobgraph.new() - print(gh) -end - diff --git a/tests/modules/async/run_callback.lua b/tests/modules/async/run_callback.lua new file mode 100644 index 000000000..21e75014b --- /dev/null +++ b/tests/modules/async/run_callback.lua @@ -0,0 +1,19 @@ +import("core.base.scheduler") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test callback ====================================") + local t = os.mclock() + runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua new file mode 100644 index 000000000..bcf5be2fd --- /dev/null +++ b/tests/modules/async/run_jobgraph.lua @@ -0,0 +1,28 @@ +import("core.base.scheduler") +import("async.jobgraph") +import("async.runjobs") + +function _jobfunc(job, opt) + print("%s: run job (%s)", scheduler.co_running(), job.name) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%s) end, progress: %s, dt: %d ms", scheduler.co_running(), job.name, opt.progress, dt) +end + +function main() + print("==================================== test jobpool ====================================") + local jobs = jobgraph.new() + jobs:add_job("job/root", _jobfunc) + for i = 1, 3 do + jobs:add_job("job/" .. i, _jobfunc) + for j = 1, 50 do + jobs:add_job("job/" .. i .. "/" .. j, _jobfunc) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/run_jobpool.lua b/tests/modules/async/run_jobpool.lua new file mode 100644 index 000000000..5de54e36c --- /dev/null +++ b/tests/modules/async/run_jobpool.lua @@ -0,0 +1,28 @@ +import("core.base.scheduler") +import("private.async.jobpool") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test jobpool ====================================") + local jobs = jobpool.new() + local root = jobs:addjob("job/root", _jobfunc) + for i = 1, 3 do + local job = jobs:addjob("job/" .. i, _jobfunc, {rootjob = root}) + for j = 1, 50 do + jobs:addjob("job/" .. i .. "/" .. j, _jobfunc, {rootjob = job}) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/runjobs.lua b/tests/modules/async/runjobs.lua deleted file mode 100644 index 4e2a98cc0..000000000 --- a/tests/modules/async/runjobs.lua +++ /dev/null @@ -1,43 +0,0 @@ -import("core.base.scheduler") -import("private.async.jobpool") -import("async.runjobs") - -function _jobfunc(index, total, opt) - print("%s: run job (%d/%d)", scheduler.co_running(), index, total) - local dt = os.mclock() - os.sleep(1000) - dt = os.mclock() - dt - print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) -end - -function main() - - -- test callback - print("==================================== test callback ====================================") - local t = os.mclock() - runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) - - -- test jobs - print("==================================== test jobs ====================================") - local jobs = jobpool.new() - local root = jobs:addjob("job/root", function (index, total, opt) - _jobfunc(index, total, opt) - end) - for i = 1, 3 do - local job = jobs:addjob("job/" .. i, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = root}) - for j = 1, 50 do - jobs:addjob("job/" .. i .. "/" .. j, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = job}) - end - end - t = os.mclock() - runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) -end - diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3fd36194a..30513ae56 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -31,6 +31,21 @@ function jobgraph:jobs() return self._jobs end +-- add a job to the jobgraph +-- +-- e.g. +-- jobgraph:add_job("xxx", function (job, opt) +-- end) +-- +-- @param name the job name +-- @param run the job run command/script +-- @param opt the job options +-- +function jobgraph:add_job(name, run, opt) + local job = {name = name, run = run, opt = opt} + self:jobs():insert(job) +end + -- tostring function jobgraph:__tostring() return string.format("", self:jobs():size()) -- cgit v1.3.1 From f8346ab6e40bdcf9506f1400f74860bbcea764ff Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:23:55 +0800 Subject: improve jobgraph tests --- tests/modules/async/run_jobgraph.lua | 13 ++++++----- xmake/modules/async/jobgraph.lua | 44 ++++++++++++++++++++++++++++++++---- xmake/modules/async/runjobs.lua | 5 ++++ 3 files changed, 51 insertions(+), 11 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index bcf5be2fd..ec9c9175f 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -2,22 +2,23 @@ import("core.base.scheduler") import("async.jobgraph") import("async.runjobs") -function _jobfunc(job, opt) - print("%s: run job (%s)", scheduler.co_running(), job.name) +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) local dt = os.mclock() os.sleep(1000) dt = os.mclock() - dt - print("%s: run job (%s) end, progress: %s, dt: %d ms", scheduler.co_running(), job.name, opt.progress, dt) + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) end function main() print("==================================== test jobpool ====================================") local jobs = jobgraph.new() - jobs:add_job("job/root", _jobfunc) + jobs:add("job/root", _jobfunc) for i = 1, 3 do - jobs:add_job("job/" .. i, _jobfunc) + jobs:add("job/" .. i, _jobfunc) for j = 1, 50 do - jobs:add_job("job/" .. i .. "/" .. j, _jobfunc) + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + jobs:add_deps("job/" .. i .. "/" .. j, "job/" .. i, "job/root") end end t = os.mclock() diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 30513ae56..313bc2d11 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -24,7 +24,16 @@ import("core.base.list") import("core.base.graph") -- define module -local jobgraph = jobgraph or object {_init = {"_jobs", "_graph"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_graph", "_dirty"}} + +-- remove the given job from the job queue +function jobqueue:remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() +end -- get jobs function jobgraph:jobs() @@ -34,24 +43,49 @@ end -- add a job to the jobgraph -- -- e.g. --- jobgraph:add_job("xxx", function (job, opt) +-- jobgraph:add("xxx", function (index, total, opt) -- end) -- -- @param name the job name -- @param run the job run command/script -- @param opt the job options -- -function jobgraph:add_job(name, run, opt) +function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} self:jobs():insert(job) + self._dirty = true +end + +-- remove a given job +function jobgraph:remove(name) + self._dirty = true +end + +-- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +function jobgraph:add_deps(...) + local deps = table.pack(...) +end + +-- add jog group +function jobgraph:add_group(name, callback) +end + +-- build a job queue +function jobgraph:build() + return jobqueue {self} +end + +-- get job size +function jobgraph:size() + return self:jobs():size() end -- tostring function jobgraph:__tostring() - return string.format("", self:jobs():size()) + return string.format("", self:size()) end -- new a jobgraph function new() - return jobgraph {list.new(), graph.new(true)} + return jobgraph {list.new(), graph.new(true), false} end diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index 0c2a6f366..959eb1e77 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -64,6 +64,11 @@ function main(name, jobs, opt) local group_name = name local jobs_cb = type(jobs) == "function" and jobs or nil assert(timeout < 60000, "runjobs: invalid timeout!") + + -- build jobs queue + if jobs.build then + jobs = jobs:build() + end assert(jobs, "runjobs: no jobs!") -- show waiting tips? -- cgit v1.3.1 From 08fade0d0b92ed766da0f752d375eda1126291f8 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:29:19 +0800 Subject: add build job queue stub --- xmake/modules/async/jobgraph.lua | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 313bc2d11..612e56dee 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,7 +25,23 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_jobs", "_graph", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_jobs", "_size", "_deps", "_dirty"}} + +-- build the job queue +function jobqueue:_build() + local graph = self._jobgraph + -- TODO + print("build job queue") +end + +-- update the job queue +function jobqueue:_update() + local graph = self._jobgraph + if graph._dirty then + self:_build() + graph._dirty = false + end +end -- remove the given job from the job queue function jobqueue:remove(job) @@ -33,11 +49,9 @@ end -- get a free job from the job queue function jobqueue:getfree() -end --- get jobs -function jobgraph:jobs() - return self._jobs + -- update the job queue first + self:_update() end -- add a job to the jobgraph @@ -51,23 +65,37 @@ end -- @param opt the job options -- function jobgraph:add(name, run, opt) - local job = {name = name, run = run, opt = opt} - self:jobs():insert(job) - self._dirty = true + local jobs = self._jobs + if not jobs[name] then + local job = {name = name, run = run, opt = opt} + jobs[name] = job + self._size = self._size + 1 + self._dirty = true + end end -- remove a given job function jobgraph:remove(name) - self._dirty = true + local jobs = self._jobs + if jobs[name] then + assert(self._size > 0) + jobs[name] = nil + self._size = self._size - 1 + self._dirty = true + end end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) + -- TODO local deps = table.pack(...) + self._dirty = true end -- add jog group function jobgraph:add_group(name, callback) + -- TODO + self._dirty = true end -- build a job queue @@ -75,9 +103,14 @@ function jobgraph:build() return jobqueue {self} end +-- get jobs +function jobgraph:jobs() + return self._jobs +end + -- get job size function jobgraph:size() - return self:jobs():size() + return self._size end -- tostring @@ -87,5 +120,5 @@ end -- new a jobgraph function new() - return jobgraph {list.new(), graph.new(true), false} + return jobgraph {{}, 0, graph.new(true), false} end -- cgit v1.3.1 From a588ee1c3f407cdfcbe5bc9753a5be1b7ca72e7e Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:30:01 +0800 Subject: fix runjobs --- xmake/modules/async/runjobs.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index 959eb1e77..d2af72843 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -66,7 +66,7 @@ function main(name, jobs, opt) assert(timeout < 60000, "runjobs: invalid timeout!") -- build jobs queue - if jobs.build then + if type(jobs) == "table" and jobs.build then jobs = jobs:build() end assert(jobs, "runjobs: no jobs!") -- cgit v1.3.1 From bc4f14679956d6eac336cb23d72e5e91c189a5bf Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 19 Mar 2025 23:35:27 +0800 Subject: add deps for jobgraph --- tests/modules/async/run_jobgraph.lua | 2 +- xmake/modules/async/jobgraph.lua | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index ec9c9175f..a3e1edeb3 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -11,7 +11,7 @@ function _jobfunc(index, total, opt) end function main() - print("==================================== test jobpool ====================================") + print("==================================== test jobgraph ====================================") local jobs = jobgraph.new() jobs:add("job/root", _jobfunc) for i = 1, 3 do diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 612e56dee..684b7d7d8 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,7 +25,7 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_jobs", "_size", "_deps", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_deps", "_dirty"}} -- build the job queue function jobqueue:_build() @@ -87,9 +87,21 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) - -- TODO - local deps = table.pack(...) - self._dirty = true + local prev + local dirty + local jobs = self._jobs + local deps = self._deps + for _, name in ipairs(table.pack(...)) do + local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) + if prev then + deps:add_edge(prev, curr) + dirty = true + end + prev = curr + end + if dirty then + self._dirty = true + end end -- add jog group @@ -108,6 +120,11 @@ function jobgraph:jobs() return self._jobs end +-- get jobgraph name +function jobgraph:name() + return self._name +end + -- get job size function jobgraph:size() return self._size @@ -115,10 +132,10 @@ end -- tostring function jobgraph:__tostring() - return string.format("", self:size()) + return string.format("", self:name() or "anonymous", self:size()) end -- new a jobgraph -function new() - return jobgraph {{}, 0, graph.new(true), false} +function new(name) + return jobgraph {name, {}, 0, graph.new(true), false} end -- cgit v1.3.1 From 79522ada6117fb6bc87dc5cc7a0a88cfde5e12fc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 00:43:39 +0800 Subject: rename to dag --- xmake/modules/async/jobgraph.lua | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 684b7d7d8..ecd5ff7f1 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,13 +25,16 @@ import("core.base.graph") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_deps", "_dirty"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} -- build the job queue function jobqueue:_build() local graph = self._jobgraph - -- TODO - print("build job queue") + local dag = graph._dag + local queue = dag:topological_sort() + for _, v in ipairs(queue) do + print(v.name) + end end -- update the job queue @@ -89,12 +92,12 @@ end function jobgraph:add_deps(...) local prev local dirty + local dag = self._dag local jobs = self._jobs - local deps = self._deps for _, name in ipairs(table.pack(...)) do local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) if prev then - deps:add_edge(prev, curr) + dag:add_edge(prev, curr) dirty = true end prev = curr -- cgit v1.3.1 From 30ca6c18a4070e5f699f5fc61bf5fca475bcd67b Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 00:47:02 +0800 Subject: build job queue --- xmake/modules/async/jobgraph.lua | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ecd5ff7f1..458c20f5a 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -24,16 +24,17 @@ import("core.base.list") import("core.base.graph") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} -- build the job queue function jobqueue:_build() local graph = self._jobgraph local dag = graph._dag - local queue = dag:topological_sort() - for _, v in ipairs(queue) do - print(v.name) + local queue = self._queue + queue:clear() + for _, job in ipairs(dag:topological_sort()) do + queue:insert(job) end end @@ -48,13 +49,18 @@ end -- remove the given job from the job queue function jobqueue:remove(job) + local queue = self._queue + queue:remove(job) end -- get a free job from the job queue function jobqueue:getfree() - - -- update the job queue first self:_update() + + local queue = self._queue + if queue:empty() then + return + end end -- add a job to the jobgraph @@ -115,7 +121,7 @@ end -- build a job queue function jobgraph:build() - return jobqueue {self} + return jobqueue {self, list.new()} end -- get jobs -- cgit v1.3.1 From 2277ac87ba989b2b4004c07627235b48760579cc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:37:13 +0800 Subject: improve graph --- xmake/core/base/graph.lua | 50 +++++++++++++++++++++++++++------------- xmake/modules/async/jobgraph.lua | 13 ++++++++--- 2 files changed, 44 insertions(+), 19 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 80d4566ec..1fb6b01dd 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -24,11 +24,11 @@ local object = require("base/object") -- define module local graph = graph or object { _init = {"_directed"} } {true} -local edge = edge or object { _init = {"_from", "_to", "_weight"} } +local edge = edge or object { _init = {"_from", "_to"} } -- new edge, from -> to -function edge.new(from, to, weight) - return edge {from, to, weight or 1.0} +function edge.new(from, to) + return edge {from, to} end function edge:from() @@ -47,8 +47,8 @@ function edge:other(v) end end -function edge:weight() - return self._weight +function edge:__tostring() + return string.format("", self:from(), self:to()) end -- clear graph @@ -56,6 +56,7 @@ function graph:clear() self._vertices = {} self._edges = {} self._adjacent_edges = {} + self._edges_map = {} end -- is empty? @@ -98,13 +99,19 @@ function graph:remove_vertex(v) end end) if contains then + self._edges_map[v] = nil self._adjacent_edges[v] = nil -- remove the adjacent edge with this vertex in the other vertices if not self:is_directed() then for _, w in ipairs(self:vertices()) do local edges = self:adjacent_edges(w) if edges then - table.remove_if(edges, function (_, e) return e:other(w) == v end) + table.remove_if(edges, function (_, e) + if e:other(w) == v then + self._edges_map[w] = nil + return true + end + end) end end end @@ -189,8 +196,8 @@ function graph:edges() end -- add edge -function graph:add_edge(from, to, weight) - local e = edge.new(from, to, weight) +function graph:add_edge(from, to) + local e = edge.new(from, to) if not self:has_vertex(from) then table.insert(self._vertices, from) self._adjacent_edges[from] = {} @@ -199,11 +206,16 @@ function graph:add_edge(from, to, weight) table.insert(self._vertices, to) self._adjacent_edges[to] = {} end + local edges_map = self._edges_map + edges_map[from] = edges_map[from] or {} + edges_map[from][to] = true if self:is_directed() then - table.insert(self._adjacent_edges[e:from()], e) + table.insert(self._adjacent_edges[from], e) else - table.insert(self._adjacent_edges[e:from()], e) - table.insert(self._adjacent_edges[e:to()], e) + table.insert(self._adjacent_edges[from], e) + table.insert(self._adjacent_edges[to], e) + edges_map[to] = edges_map[to] or {} + edges_map[to][from] = true end table.insert(self._edges, e) end @@ -212,9 +224,15 @@ end function graph:has_edge(from, to) local edges = self:adjacent_edges(from) if edges then - for _, e in ipairs(edges) do - if e:to() == to then - return true + local edges_map = self._edges_map + local from_map = edges_map[from] + if from_map and from_map[to] then + return true + else + for _, e in ipairs(edges) do + if e:to() == to then + return true + end end end end @@ -228,7 +246,7 @@ function graph:clone() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:from(), e:to(), e:weight()) + gh:add_edge(e:from(), e:to()) end end end @@ -245,7 +263,7 @@ function graph:reverse() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:to(), e:from(), e:weight()) + gh:add_edge(e:to(), e:from()) end end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 458c20f5a..bfeee3863 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -33,7 +33,9 @@ function jobqueue:_build() local dag = graph._dag local queue = self._queue queue:clear() + -- TODO find cycle for _, job in ipairs(dag:topological_sort()) do + print("build job", job.name) queue:insert(job) end end @@ -86,9 +88,12 @@ end -- remove a given job function jobgraph:remove(name) local jobs = self._jobs - if jobs[name] then + local job = jobs[name] + local dag = self._dag + if job then assert(self._size > 0) jobs[name] = nil + dag:remove_vertex(job) self._size = self._size - 1 self._dirty = true end @@ -103,8 +108,10 @@ function jobgraph:add_deps(...) for _, name in ipairs(table.pack(...)) do local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) if prev then - dag:add_edge(prev, curr) - dirty = true + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) + dirty = true + end end prev = curr end -- cgit v1.3.1 From 8954ac2acaf229a1007b37f4caadfb10a9d78fa4 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:39:52 +0800 Subject: check cirular dependencies --- xmake/modules/async/jobgraph.lua | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index bfeee3863..31a041911 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -32,8 +32,20 @@ function jobqueue:_build() local graph = self._jobgraph local dag = graph._dag local queue = self._queue + + -- check circular dependencies + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, job in ipairs(cycle) do + table.insert(names, job.name) + end + table.insert(names, names[1]) + raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) + end + + -- build job queue queue:clear() - -- TODO find cycle for _, job in ipairs(dag:topological_sort()) do print("build job", job.name) queue:insert(job) -- cgit v1.3.1 From 5b57870c2854f96c9fd2aebb6e9fe538b9a27b8c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 22:48:36 +0800 Subject: add job deps --- xmake/core/base/graph.lua | 9 +++++++-- xmake/modules/async/jobgraph.lua | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 1fb6b01dd..bf6b724ce 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -119,7 +119,8 @@ function graph:remove_vertex(v) end -- topological sort -function graph:topological_sort() +function graph:topological_sort(opt) + opt = opt or {} local visited = {} for _, v in ipairs(self:vertices()) do visited[v] = false @@ -143,7 +144,11 @@ function graph:topological_sort() dfs(v) end end - return table.reverse(order_vertices) + if opt.reverse then + return order_vertices + else + return table.reverse(order_vertices) + end end -- find cycle diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 31a041911..9bf98f7b5 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -22,11 +22,25 @@ import("core.base.object") import("core.base.list") import("core.base.graph") +import("core.base.hashset") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} +-- add job dependency +function jobqueue:_add_dep(job, dep) + job._deps = job._deps or hashset.new() + job._deps:insert(dep) + + local parents = dep._parents + if not parents then + parents = {} + dep._parents = parents + end + table.insert(parents, job) +end + -- build the job queue function jobqueue:_build() local graph = self._jobgraph @@ -46,10 +60,17 @@ function jobqueue:_build() -- build job queue queue:clear() - for _, job in ipairs(dag:topological_sort()) do - print("build job", job.name) + for _, job in ipairs(dag:topological_sort({reverse = true})) do + job._deps = nil + job._parents = nil queue:insert(job) end + + -- build job dependencies + for _, e in ipairs(dag:edges()) do + self:_add_dep(e:from(), e:to()) + print("%s -> %s", e:from().name, e:to().name) + end end -- update the job queue @@ -64,7 +85,9 @@ end -- remove the given job from the job queue function jobqueue:remove(job) local queue = self._queue + print("remove", job.name) queue:remove(job) + -- TODO remove deps end -- get a free job from the job queue @@ -75,6 +98,12 @@ function jobqueue:getfree() if queue:empty() then return end + + -- TODO + for job in queue:ritems() do + print("get free job", job.name) + return job + end end -- add a job to the jobgraph -- cgit v1.3.1 From a278d3bd3e74027e7fd1f878c7d8c991caf1a75f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 23:16:51 +0800 Subject: improve to find cycle --- tests/modules/graph/test.lua | 3 +++ xmake/core/base/graph.lua | 19 ++++++++++++---- xmake/core/tool/builder.lua | 11 ++++++---- xmake/modules/async/jobgraph.lua | 25 +++++++++++----------- .../modules/modules_support/dependency_scanner.lua | 25 +++++++++++----------- 5 files changed, 51 insertions(+), 32 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 63095be12..4bbefe069 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -52,5 +52,8 @@ function test_find_cycle(t) end local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) + + local _, has_cycle = dag:topological_sort() + t:require(has_cycle) end diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index bf6b724ce..3b88a642a 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -125,29 +125,40 @@ function graph:topological_sort(opt) for _, v in ipairs(self:vertices()) do visited[v] = false end + local in_stack = {} local order_vertices = {} local function dfs(v) visited[v] = true + in_stack[v] = true local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do local w = e:other(v) if not visited[w] then - dfs(w) + if dfs(w) then + return true + end + elseif in_stack[w] then + return true end end end + in_stack[v] = false table.insert(order_vertices, v) end + local has_cycle = false for _, v in ipairs(self:vertices()) do if not visited[v] then - dfs(v) + if dfs(v) then + has_cycle = true + break + end end end if opt.reverse then - return order_vertices + return order_vertices, has_cycle else - return table.reverse(order_vertices) + return table.reverse(order_vertices), has_cycle end end diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index 2bf4efc1f..b0f802237 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -668,11 +668,14 @@ function builder:_sort_links_of_items(items, opt) gh:add_edge(k, v) end if not gh:empty() then - local cycle = gh:find_cycle() - if cycle then - utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + local has_cycle + links, has_cycle = gh:topological_sort() + if has_cycle then + local cycle = gh:find_cycle() + if cycle then + utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + end end - links = gh:topological_sort() end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 9bf98f7b5..6d96ff0ff 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -47,20 +47,21 @@ function jobqueue:_build() local dag = graph._dag local queue = self._queue - -- check circular dependencies - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, job in ipairs(cycle) do - table.insert(names, job.name) - end - table.insert(names, names[1]) - raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) - end - -- build job queue queue:clear() - for _, job in ipairs(dag:topological_sort({reverse = true})) do + local order_jobs, has_cycle = dag:topological_sort({reverse = true}) + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, job in ipairs(cycle) do + table.insert(names, job.name) + end + table.insert(names, names[1]) + raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) + end + end + for _, job in ipairs(order_jobs) do job._deps = nil job._parents = nil queue:insert(job) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 9d552d0fd..df8c85441 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -222,7 +222,7 @@ function _generate_dependencies(target, sourcebatch, opt) local changed = false if opt.batchjobs then local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) + runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed end, {comax = jobs, total = #sourcebatch.sourcefiles}) @@ -415,19 +415,20 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, objectfile in ipairs(cycle) do - local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, objectfile in ipairs(cycle) do + local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + table.insert(names, name or cppfile) + end + local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) table.insert(names, name or cppfile) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) - table.insert(names, name or cppfile) - raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - - local objectfiles_sorted = table.reverse(dag:topological_sort()) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then @@ -465,7 +466,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) end end end - if insert then + if insert then table.insert(build_objectfiles, objectfile) table.insert(link_objectfiles, objectfile) elseif external and not external.from_moduleonly then -- cgit v1.3.1 From ce5fb205bbebb83fe6842b00858210f9d4256148 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 22:37:17 +0800 Subject: use kahn algorithm for graph by default --- xmake/core/base/graph.lua | 91 +++++++++++++++++++++++++++++++++++++--- xmake/modules/async/jobgraph.lua | 24 +---------- 2 files changed, 87 insertions(+), 28 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 3b88a642a..06324c0a0 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -118,9 +118,8 @@ function graph:remove_vertex(v) end end --- topological sort -function graph:topological_sort(opt) - opt = opt or {} +-- topological sort, use DFS algorithom +function graph:_topological_sort_dfs() local visited = {} for _, v in ipairs(self:vertices()) do visited[v] = false @@ -155,10 +154,90 @@ function graph:topological_sort(opt) end end end - if opt.reverse then - return order_vertices, has_cycle + return table.reverse(order_vertices), has_cycle +end + +-- topological sort, use Kahn's algorithm +function graph:_topological_sort_kahn() + + -- calculate in-degree for each vertex + local in_degree = {} + for _, v in ipairs(self:vertices()) do + in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + in_degree[w] = (in_degree[w] or 0) + 1 + end + end + end + end + + -- queue of vertices with no incoming edges (no dependencies) + local queue = {} + for _, v in ipairs(self:vertices()) do + if in_degree[v] == 0 then + table.insert(queue, v) + end + end + + -- result list for topologically sorted vertices + local order_vertices = {} + + -- process queue + while #queue > 0 do + -- remove a vertex with no incoming edges + local v = table.remove(queue, 1) + table.insert(order_vertices, v) + + -- for each outgoing edge, remove it and update in-degrees + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + in_degree[w] = in_degree[w] - 1 + -- if in-degree becomes zero, add to queue + if in_degree[w] == 0 then + table.insert(queue, w) + end + end + end + end + end + + -- if we couldn't process all vertices, there must be a cycle + local has_cycle = #order_vertices ~= #self:vertices() + + return order_vertices, has_cycle +end + +-- topological sort (default: Kahn's algorithm) +-- +-- @param opt the options, we can use `{algorithm = "dfs/kahn"}` to select sort algorithm, +-- and the Kahn is the default algorithm. +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- it will return {c, b, a} +function graph:topological_sort(opt) + opt = opt or {} + if not self:is_directed() then + return + end + if opt.algorithm == "dfs" then + return self:_topological_sort_dfs() else - return table.reverse(order_vertices), has_cycle + return self:_topological_sort_kahn() end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 6d96ff0ff..ca8725b28 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -28,19 +28,6 @@ import("core.base.hashset") local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} --- add job dependency -function jobqueue:_add_dep(job, dep) - job._deps = job._deps or hashset.new() - job._deps:insert(dep) - - local parents = dep._parents - if not parents then - parents = {} - dep._parents = parents - end - table.insert(parents, job) -end - -- build the job queue function jobqueue:_build() local graph = self._jobgraph @@ -49,7 +36,7 @@ function jobqueue:_build() -- build job queue queue:clear() - local order_jobs, has_cycle = dag:topological_sort({reverse = true}) + local order_jobs, has_cycle = dag:topological_sort() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -62,16 +49,9 @@ function jobqueue:_build() end end for _, job in ipairs(order_jobs) do - job._deps = nil - job._parents = nil + print("insert", job.name) queue:insert(job) end - - -- build job dependencies - for _, e in ipairs(dag:edges()) do - self:_add_dep(e:from(), e:to()) - print("%s -> %s", e:from().name, e:to().name) - end end -- update the job queue -- cgit v1.3.1 From 741da62196bcb64c88e386f1a769462211c63b4e Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:03:29 +0800 Subject: add queue and improve graph --- tests/modules/graph/test.lua | 8 +- tests/modules/queue/test.lua | 35 ++++ xmake/core/base/graph.lua | 216 +++++++++++++++++++-- xmake/core/base/queue.lua | 125 ++++++++++++ .../sandbox/modules/import/core/base/queue.lua | 22 +++ xmake/core/tool/builder.lua | 2 +- xmake/modules/async/jobgraph.lua | 54 ++---- xmake/modules/cli/amalgamate.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 3 +- 9 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 tests/modules/queue/test.lua create mode 100644 xmake/core/base/queue.lua create mode 100644 xmake/core/sandbox/modules/import/core/base/queue.lua (limited to 'xmake/modules') diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 4bbefe069..33e1f3fa7 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -1,6 +1,6 @@ import("core.base.graph") -function test_topological_sort(t) +function test_topo_sort(t) local edges = { {0, 5}, {0, 2}, @@ -18,7 +18,7 @@ function test_topological_sort(t) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local order_path = dag:topological_sort() + local order_path = dag:topo_sort() local orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -28,7 +28,7 @@ function test_topological_sort(t) end dag = dag:reverse() - order_path = dag:topological_sort() + order_path = dag:topo_sort() orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -53,7 +53,7 @@ function test_find_cycle(t) local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) - local _, has_cycle = dag:topological_sort() + local _, has_cycle = dag:topo_sort() t:require(has_cycle) end diff --git a/tests/modules/queue/test.lua b/tests/modules/queue/test.lua new file mode 100644 index 000000000..b577e04c9 --- /dev/null +++ b/tests/modules/queue/test.lua @@ -0,0 +1,35 @@ +import("core.base.queue") + +function test_push(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + t:are_equal(d:first(), 1) + t:are_equal(d:last(), 5) + local idx = 1 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + +function test_pop(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + d:pop() + t:are_equal(d:first(), 2) + t:are_equal(d:last(), 5) + local idx = 2 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index ffcd815c5..448eebb88 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -19,9 +19,10 @@ -- -- load modules -local table = require("base/table") -local list = require("base/list") -local object = require("base/object") +local table = require("base/table") +local queue = require("base/queue") +local object = require("base/object") +local hashset = require("base/hashset") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -58,6 +59,9 @@ function graph:clear() self._edges = {} self._adjacent_edges = {} self._edges_map = {} + + -- clear partial topological sort state + self:partial_topo_sort_reset() end -- is empty? @@ -116,10 +120,171 @@ function graph:remove_vertex(v) end end end + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() + end +end + +-- check if there's a cycle in the remaining unprocessed nodes +function graph:_check_cycle_in_remaining() + -- if all remaining nodes have in-degree > 0, we have a cycle + if self._topo_remaining_count > 0 and self._topo_remaining_count == self._topo_non_zero_indegree_count then + self._topo_has_cycle = true + return true end + return false end --- topological sort, use Kahn's algorithm +-- reset partial topological sort state +function graph:partial_topo_sort_reset() + self._topo_in_progress = false + self._topo_in_degree = nil + self._topo_queue = nil + self._topo_processed = nil + self._topo_has_cycle = nil + self._topo_remaining_count = nil + self._topo_non_zero_indegree_count = nil +end + +-- get next batch of nodes in topological order with limit +-- +-- @param limit the maximum number of nodes to return +-- @return array of nodes with zero in-degree, empty when complete +-- @return has_cycle indicates if a cycle was detected +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- local batch1, has_cycle = g:partial_topo_sort_next(1) -- returns {c} +-- local batch2, has_cycle = g:partial_topo_sort_next(1) -- returns {b} +-- local batch3, has_cycle = g:partial_topo_sort_next(1) -- returns {a} +-- local batch4, has_cycle = g:partial_topo_sort_next(1) -- returns {} (empty, all done) +-- +function graph:partial_topo_sort_next(limit) + if not self:is_directed() then + return {}, false + end + + limit = limit or math.huge + + -- check if we already detected a cycle + if self._topo_has_cycle then + return {}, true + end + + -- initialize topological sort state if not already in progress + if not self._topo_in_progress then + -- calculate in-degree for each vertex + self._topo_in_degree = {} + for _, v in ipairs(self:vertices()) do + self._topo_in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = (self._topo_in_degree[w] or 0) + 1 + end + end + end + end + + -- initialize queue with vertices that have no incoming edges + self._topo_queue = queue.new() + for _, v in ipairs(self:vertices()) do + if self._topo_in_degree[v] == 0 then + self._topo_queue:push(v) + end + end + + -- track processed vertices + self._topo_processed = hashset.new() + self._topo_in_progress = true + + -- track counts for efficient cycle detection + self._topo_remaining_count = #self:vertices() + self._topo_non_zero_indegree_count = self._topo_remaining_count - self._topo_queue:size() + + -- quick cycle detection: if no nodes have zero in-degree, we have a cycle + if self._topo_queue:empty() and self._topo_remaining_count > 0 then + self._topo_has_cycle = true + return {}, true + end + end + + -- return empty batch if queue is empty (all processed or cycle detected) + if self._topo_queue:empty() then + -- check if all vertices were processed + local processed_count = self._topo_processed:size() + self._topo_has_cycle = processed_count ~= #self:vertices() + + -- if this is the first call and we detect a cycle, mark as complete + if processed_count == 0 then + self._topo_in_progress = false + end + + return {}, self._topo_has_cycle + end + + -- collect up to 'limit' nodes with zero in-degree + local batch = {} + while not self._topo_queue:empty() and #batch < limit do + local v = self._topo_queue:pop() + table.insert(batch, v) + self._topo_processed:insert(v) + self._topo_remaining_count = self._topo_remaining_count - 1 + end + + -- update in-degrees based on the nodes in this batch + for _, v in ipairs(batch) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = self._topo_in_degree[w] - 1 + + -- update non-zero in-degree count + if self._topo_in_degree[w] == 0 then + self._topo_non_zero_indegree_count = self._topo_non_zero_indegree_count - 1 + + -- if in-degree becomes zero, add to queue for next batch + if not self._topo_processed:has(w) then + self._topo_queue:push(w) + end + end + end + end + end + end + + -- early cycle detection - if all remaining nodes have in-degree > 0 + if self:_check_cycle_in_remaining() then + return batch, true + end + + -- if queue is now empty and all vertices processed, reset state + if self._topo_queue:empty() then + local processed_count = self._topo_processed:size() + if processed_count == #self:vertices() then + self._topo_in_progress = false + else + -- if queue is empty but we still have unprocessed nodes, we have a cycle + self._topo_has_cycle = true + end + end + + return batch, self._topo_has_cycle +end + +-- topological sort, use kahn's algorithm -- -- e.g. -- @@ -127,8 +292,35 @@ end -- add_edge(b, c) -- b depend on c -- -- it will return {c, b, a} -function graph:topological_sort(opt) - opt = opt or {} +--[[ +function graph:topo_sort() + if not self:is_directed() then + return + end + + -- reset partial sort state to ensure we start fresh + self:partial_topo_sort_reset() + + local order_vertices = {} + local batch_size = math.huge -- no limit, get all at once + + -- get all nodes in one go + local batch, has_cycle = self:partial_topo_sort_next(batch_size) + while #batch > 0 do + for _, v in ipairs(batch) do + table.insert(order_vertices, v) + end + batch, has_cycle = self:partial_topo_sort_next(batch_size) + + -- quick exit if cycle is detected + if has_cycle then + break + end + end + + return order_vertices, has_cycle +end]] +function graph:topo_sort() if not self:is_directed() then return end @@ -153,10 +345,10 @@ function graph:topological_sort(opt) end -- queue of vertices with no incoming edges (no dependencies) - local queue = list.new() + local queue = queue.new() for _, v in ipairs(self:vertices()) do if in_degree[v] == 0 then - queue:insert(v) + queue:push(v) end end @@ -166,7 +358,7 @@ function graph:topological_sort(opt) -- process queue while not queue:empty() do -- remove a vertex with no incoming edges - local v = queue:remove_first() + local v = queue:pop() table.insert(order_vertices, v) -- for each outgoing edge, remove it and update in-degrees @@ -178,7 +370,7 @@ function graph:topological_sort(opt) in_degree[w] = in_degree[w] - 1 -- if in-degree becomes zero, add to queue if in_degree[w] == 0 then - queue:insert(w) + queue:push(w) end end end @@ -263,6 +455,9 @@ function graph:add_edge(from, to) edges_map[to][from] = true end table.insert(self._edges, e) + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() end -- has the given edge? @@ -340,4 +535,3 @@ end -- return module: graph return graph - diff --git a/xmake/core/base/queue.lua b/xmake/core/base/queue.lua new file mode 100644 index 000000000..dbdc1d87a --- /dev/null +++ b/xmake/core/base/queue.lua @@ -0,0 +1,125 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- load modules +local object = require("base/object") + +-- define module +local queue = queue or object {_init = {"_first", "_last"}} {1, 0} + +-- clear queue +function queue:clear() + self._first = 1 + self._last = 0 +end + +-- push item to queue +function queue:push(item) + local last = self._last + 1 + self._last = last + self[last] = item +end + +-- pop item from queue +function queue:pop() + local first = self._first + if first > self._last then + return nil + end + + local value = self[first] + self[first] = nil + self._first = first + 1 + return value +end + +-- get queue size +function queue:size() + return self._last - self._first + 1 +end + +-- is queue empty? +function queue:empty() + return self._first > self._last +end + +-- peek the first item of queue +function queue:first() + if self._first > self._last then + return nil + end + return self[self._first] +end + +-- peek the last item of queue +function queue:last() + if self._first > self._last then + return nil + end + return self[self._last] +end + +-- iterator for all items (forward) +-- +-- e.g. +-- +-- for item in queue:items() do +-- print(item) +-- end +-- +function queue:items() + local index = self._first - 1 + local last = self._last + return function() + index = index + 1 + if index <= last then + return self[index] + end + end +end + +-- iterator for all items (reverse) +function queue:ritems() + local index = self._last + 1 + local first = self._first + return function() + index = index - 1 + if index >= first then + return self[index] + end + end +end + +-- clone queue +function queue:clone() + local q = queue.new() + for i = self._first, self._last do + q:push(self[i]) + end + return q +end + +-- new queue +function queue.new() + return queue() +end + +-- return module: queue +return queue diff --git a/xmake/core/sandbox/modules/import/core/base/queue.lua b/xmake/core/sandbox/modules/import/core/base/queue.lua new file mode 100644 index 000000000..aab0a8214 --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/base/queue.lua @@ -0,0 +1,22 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- return module +return require("base/queue") diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index b0f802237..0e85911eb 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -669,7 +669,7 @@ function builder:_sort_links_of_items(items, opt) end if not gh:empty() then local has_cycle - links, has_cycle = gh:topological_sort() + links, has_cycle = gh:topo_sort() if has_cycle then local cycle = gh:find_cycle() if cycle then diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ca8725b28..4a0dac110 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,18 +25,20 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} --- build the job queue -function jobqueue:_build() +-- remove the given job from the job queue +function jobqueue:remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() local graph = self._jobgraph local dag = graph._dag - local queue = self._queue -- build job queue - queue:clear() - local order_jobs, has_cycle = dag:topological_sort() + local order_jobs, has_cycle = dag:partial_topo_sort_next(1) if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -48,42 +50,8 @@ function jobqueue:_build() raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end end - for _, job in ipairs(order_jobs) do - print("insert", job.name) - queue:insert(job) - end -end - --- update the job queue -function jobqueue:_update() - local graph = self._jobgraph - if graph._dirty then - self:_build() - graph._dirty = false - end -end - --- remove the given job from the job queue -function jobqueue:remove(job) - local queue = self._queue - print("remove", job.name) - queue:remove(job) - -- TODO remove deps -end - --- get a free job from the job queue -function jobqueue:getfree() - self:_update() - - local queue = self._queue - if queue:empty() then - return - end - - -- TODO - for job in queue:ritems() do - print("get free job", job.name) - return job + if order_jobs then + return table.unwrap(order_jobs) end end @@ -150,7 +118,7 @@ end -- build a job queue function jobgraph:build() - return jobqueue {self, list.new()} + return jobqueue {self} end -- get jobs diff --git a/xmake/modules/cli/amalgamate.lua b/xmake/modules/cli/amalgamate.lua index 8c7ec6fd7..d0a5ba40c 100644 --- a/xmake/modules/cli/amalgamate.lua +++ b/xmake/modules/cli/amalgamate.lua @@ -96,7 +96,7 @@ function _generate_file(target, inputpaths, outputpath, uniqueid) _generate_include_graph(target, inputpaths, gh, {}) -- sort file paths and remove root path - local filepaths = gh:topological_sort() + local filepaths = gh:topo_sort() table.remove(filepaths, 1) -- generate amalgamate file diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index df8c85441..94c14e60d 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -415,7 +415,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + local objectfiles_sorted, has_cycle = dag:topo_sort() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -429,6 +429,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end end + objectfiles_sorted = table.reverse(objectfiles_sorted) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then -- cgit v1.3.1 From 5d6d739f00f781f91d30ffe93811128b968a151f Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:20:56 +0800 Subject: improve jobgraph --- xmake/core/base/graph.lua | 28 ++++++++-------------------- xmake/modules/async/jobgraph.lua | 31 ++++++++++--------------------- 2 files changed, 18 insertions(+), 41 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index b2d22a3c4..5d4ac1abb 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -223,28 +223,19 @@ function graph:partial_topo_sort_next(limit) end end - -- return empty batch if queue is empty (all processed or cycle detected) + local node if self._partial_topo_queue:empty() then - -- check if all vertices were processed + -- return empty node if queue is empty (all processed or cycle detected) local processed_count = self._partial_topo_processed:size() self._partial_topo_has_cycle = processed_count ~= #self:vertices() - - -- if this is the first call and we detect a cycle, mark as complete - if processed_count == 0 then - self._partial_topo_in_progress = false - end - return nil, self._partial_topo_has_cycle - end - - -- get one node with zero in-degree - local node - if not self._partial_topo_queue:empty() then + else + -- get one node with zero in-degree node = self._partial_topo_queue:pop() self._partial_topo_processed:insert(node) self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 - -- update in-degrees based on the nodes in this batch + -- update in-degrees based on the nodes in this node local edges = self:adjacent_edges(node) if edges then for _, e in ipairs(edges) do @@ -256,7 +247,7 @@ function graph:partial_topo_sort_next(limit) if self._partial_topo_in_degree[w] == 0 then self._partial_topo_non_zero_indegree_count = self._partial_topo_non_zero_indegree_count - 1 - -- if in-degree becomes zero, add to queue for next batch + -- if in-degree becomes zero, add to queue for next node if not self._partial_topo_processed:has(w) then self._partial_topo_queue:push(w) end @@ -271,13 +262,10 @@ function graph:partial_topo_sort_next(limit) return node, true end - -- if queue is now empty and all vertices processed, reset state + -- if queue is empty but we still have unprocessed nodes, we have a cycle if self._partial_topo_queue:empty() then local processed_count = self._partial_topo_processed:size() - if processed_count == #self:vertices() then - self._partial_topo_in_progress = false - else - -- if queue is empty but we still have unprocessed nodes, we have a cycle + if processed_count ~= #self:vertices() then self._partial_topo_has_cycle = true end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 4a0dac110..46b10381b 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,20 +25,17 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} +local jobqueue = jobqueue or object {_init = {"_dag"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} --- remove the given job from the job queue +-- nothing to do, we need not to remove it function jobqueue:remove(job) end -- get a free job from the job queue function jobqueue:getfree() - local graph = self._jobgraph - local dag = graph._dag - - -- build job queue - local order_jobs, has_cycle = dag:partial_topo_sort_next(1) + local dag = self._dag + local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -50,9 +47,7 @@ function jobqueue:getfree() raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end end - if order_jobs then - return table.unwrap(order_jobs) - end + return freejob end -- add a job to the jobgraph @@ -71,7 +66,6 @@ function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 - self._dirty = true end end @@ -85,14 +79,12 @@ function jobgraph:remove(name) jobs[name] = nil dag:remove_vertex(job) self._size = self._size - 1 - self._dirty = true end end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) local prev - local dirty local dag = self._dag local jobs = self._jobs for _, name in ipairs(table.pack(...)) do @@ -100,25 +92,22 @@ function jobgraph:add_deps(...) if prev then if not dag:has_edge(prev, curr) then dag:add_edge(prev, curr) - dirty = true end end prev = curr end - if dirty then - self._dirty = true - end end -- add jog group function jobgraph:add_group(name, callback) -- TODO - self._dirty = true end -- build a job queue function jobgraph:build() - return jobqueue {self} + local dag = self._dag + dag:partial_topo_sort_reset() + return jobqueue {dag} end -- get jobs @@ -143,5 +132,5 @@ end -- new a jobgraph function new(name) - return jobgraph {name, {}, 0, graph.new(true), false} + return jobgraph {name, {}, 0, graph.new(true)} end -- cgit v1.3.1 From f1a16a647593691302154af0ff592a9cdf9cda5e Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:36:14 +0800 Subject: fix remove node --- tests/modules/graph/test.lua | 3 ++ xmake/core/base/graph.lua | 83 ++++++++++++---------------------------- xmake/modules/async/jobgraph.lua | 12 +++--- 3 files changed, 34 insertions(+), 64 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index c9584a69a..1fb0bcb43 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -50,6 +50,9 @@ function test_paritail_topo_sort(t) break end table.insert(order_vertices, node) + if node then + dag:partial_topo_sort_remove(node) + end end return order_vertices, has_cycle diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index bd6c8a3cc..04b0c6573 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -132,9 +132,8 @@ function graph:partial_topo_sort_reset() self._partial_topo_in_degree = nil self._partial_topo_queue = nil self._partial_topo_processed = nil + self._partial_topo_pending = 0 self._partial_topo_has_cycle = nil - self._partial_topo_remaining_count = nil - self._partial_topo_non_zero_indegree_count = nil self._partial_topo_dirty = false end @@ -173,59 +172,44 @@ function graph:partial_topo_sort_next(limit) if not self._partial_topo_in_progress then self:_partial_topo_sort_init() self._partial_topo_in_progress = true - if self._partial_topo_has_cycle then - return nil, true - end end + -- get one node with zero in-degree local node - if self._partial_topo_queue:empty() then - -- return empty node if queue is empty (all processed or cycle detected) - local processed_count = self._partial_topo_processed:size() - self._partial_topo_has_cycle = processed_count ~= #self:vertices() - return nil, self._partial_topo_has_cycle - else - -- get one node with zero in-degree + if not self._partial_topo_queue:empty() then node = self._partial_topo_queue:pop() self._partial_topo_processed:insert(node) - self._partial_topo_remaining_count = self._partial_topo_remaining_count - 1 - - -- update in-degrees based on the nodes in this node - local edges = self:adjacent_edges(node) - if edges then - for _, e in ipairs(edges) do - if e:from() == node then - local w = e:to() - self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 + self._partial_topo_pending = self._partial_topo_pending + 1 + end - -- update non-zero in-degree count - if self._partial_topo_in_degree[w] == 0 then - self._partial_topo_non_zero_indegree_count = self._partial_topo_non_zero_indegree_count - 1 + return node, self._partial_topo_has_cycle +end - -- if in-degree becomes zero, add to queue for next node - if not self._partial_topo_processed:has(w) then - self._partial_topo_queue:push(w) - end +-- remove node and update in-degrees based on the nodes in this node +function graph:partial_topo_sort_remove(node) + if node == nil then + return + end + self._partial_topo_pending = self._partial_topo_pending - 1 + local edges = self:adjacent_edges(node) + if edges then + for _, e in ipairs(edges) do + if e:from() == node then + local w = e:to() + self._partial_topo_in_degree[w] = self._partial_topo_in_degree[w] - 1 + if self._partial_topo_in_degree[w] == 0 then + if not self._partial_topo_processed:has(w) then + self._partial_topo_queue:push(w) end end end end end - -- early cycle detection - if all remaining nodes have in-degree > 0 - if self:_check_cycle_in_remaining() then - return node, true - end - - -- if queue is empty but we still have unprocessed nodes, we have a cycle - if self._partial_topo_queue:empty() then + if self._partial_topo_queue:empty() and self._partial_topo_pending == 0 then local processed_count = self._partial_topo_processed:size() - if processed_count ~= #self:vertices() then - self._partial_topo_has_cycle = true - end + self._partial_topo_has_cycle = processed_count ~= #self:vertices() end - - return node, self._partial_topo_has_cycle end -- topological sort, use kahn's algorithm @@ -472,25 +456,6 @@ function graph:_partial_topo_sort_init() -- track processed vertices self._partial_topo_processed = hashset.new() - - -- track counts for efficient cycle detection - self._partial_topo_remaining_count = #self:vertices() - self._partial_topo_non_zero_indegree_count = self._partial_topo_remaining_count - self._partial_topo_queue:size() - - -- quick cycle detection: if no nodes have zero in-degree, we have a cycle - if self._partial_topo_queue:empty() and self._partial_topo_remaining_count > 0 then - self._partial_topo_has_cycle = true - end -end - --- check if there's a cycle in the remaining unprocessed nodes -function graph:_check_cycle_in_remaining() - -- if all remaining nodes have in-degree > 0, we have a cycle - if self._partial_topo_remaining_count > 0 and self._partial_topo_remaining_count == self._partial_topo_non_zero_indegree_count then - self._partial_topo_has_cycle = true - return true - end - return false end -- new graph diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 46b10381b..9994ac1f3 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,11 +25,13 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_dag"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph", "_dag"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} --- nothing to do, we need not to remove it +-- remove the finished job function jobqueue:remove(job) + local dag = self._dag + dag:partial_topo_sort_remove(job) end -- get a free job from the job queue @@ -37,15 +39,15 @@ function jobqueue:getfree() local dag = self._dag local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then + local names = {} local cycle = dag:find_cycle() if cycle then - local names = {} for _, job in ipairs(cycle) do table.insert(names, job.name) end table.insert(names, names[1]) - raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end + raise("%s: circular job dependency detected!\n%s", self._jobgraph, table.concat(names, "\n -> ")) end return freejob end @@ -107,7 +109,7 @@ end function jobgraph:build() local dag = self._dag dag:partial_topo_sort_reset() - return jobqueue {dag} + return jobqueue {self, dag} end -- get jobs -- cgit v1.3.1 From a260aa4d879b459aeaf3526816f1bc474485f508 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 00:23:09 +0800 Subject: add group --- xmake/modules/async/jobgraph.lua | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 9994ac1f3..ee3a6f274 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -26,7 +26,7 @@ import("core.base.hashset") -- define module local jobqueue = jobqueue or object {_init = {"_jobgraph", "_dag"}} -local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_groups"}} -- remove the finished job function jobqueue:remove(job) @@ -60,7 +60,7 @@ end -- -- @param name the job name -- @param run the job run command/script --- @param opt the job options +-- @param opt the job options, e.g. {group = "xxx"} -- function jobgraph:add(name, run, opt) local jobs = self._jobs @@ -68,6 +68,16 @@ function jobgraph:add(name, run, opt) local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 + + local group_name = opt.group + if group_name then + local groups = self._groups[group_name] + if not groups then + groups = {} + self._groups[group_name] = groups + end + table.insert(groups, job) + end end end @@ -98,11 +108,8 @@ function jobgraph:add_deps(...) end prev = curr end -end - --- add jog group -function jobgraph:add_group(name, callback) -- TODO + -- add groups jobs end -- build a job queue @@ -134,5 +141,5 @@ end -- new a jobgraph function new(name) - return jobgraph {name, {}, 0, graph.new(true)} + return jobgraph {name, {}, 0, graph.new(true), {}} end -- cgit v1.3.1 From d31fc3eb8ade62b0d80280f22329bb132ea95cfc Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 22:28:30 +0800 Subject: support for job group --- tests/modules/async/run_jobgraph.lua | 26 ++++++++++++-- xmake/modules/async/jobgraph.lua | 68 +++++++++++++++++++++++++++++------- 2 files changed, 79 insertions(+), 15 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index a3e1edeb3..bc1b5500f 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -10,8 +10,8 @@ function _jobfunc(index, total, opt) print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) end -function main() - print("==================================== test jobgraph ====================================") +function _test_basic() + print("==================================== test basic ====================================") local jobs = jobgraph.new() jobs:add("job/root", _jobfunc) for i = 1, 3 do @@ -27,3 +27,25 @@ function main() end}) end +function _test_group() + print("==================================== test group ====================================") + local jobs = jobgraph.new() + jobs:add("job/root", _jobfunc) + for i = 1, 3 do + jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) + end + end + jobs:add_deps("foo", "bar", "job/root") + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + +function main() + _test_basic() + _test_group() +end + diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ee3a6f274..edf3cfe81 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -37,6 +37,7 @@ end -- get a free job from the job queue function jobqueue:getfree() local dag = self._dag +::continue:: local freejob, has_cycle = dag:partial_topo_sort_next() if has_cycle then local names = {} @@ -49,6 +50,11 @@ function jobqueue:getfree() end raise("%s: circular job dependency detected!\n%s", self._jobgraph, table.concat(names, "\n -> ")) end + -- if it's a fake job, we need to skip it and continue to get the next job + if freejob and not freejob.run then + dag:partial_topo_sort_remove(freejob) + goto continue + end return freejob end @@ -60,23 +66,25 @@ end -- -- @param name the job name -- @param run the job run command/script --- @param opt the job options, e.g. {group = "xxx"} +-- @param opt the job options, e.g. {groups = {"xxx"}} -- function jobgraph:add(name, run, opt) + opt = opt or {} local jobs = self._jobs if not jobs[name] then local job = {name = name, run = run, opt = opt} jobs[name] = job self._size = self._size + 1 - local group_name = opt.group - if group_name then - local groups = self._groups[group_name] - if not groups then - groups = {} - self._groups[group_name] = groups + if opt.groups then + for _, group_name in ipairs(opt.groups) do + local groups = self._groups[group_name] + if not groups then + groups = {} + self._groups[group_name] = groups + end + table.insert(groups, job) end - table.insert(groups, job) end end end @@ -97,19 +105,53 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... function jobgraph:add_deps(...) local prev + local prev_is_group local dag = self._dag local jobs = self._jobs + local groups = self._groups for _, name in ipairs(table.pack(...)) do - local curr = assert(jobs[name], "job(%s) not found in jobgraph(%s)", name, self) + local curr_is_group = false + local curr = jobs[name] + if not curr then + curr = groups[name] + curr_is_group = true + end + assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then - if not dag:has_edge(prev, curr) then - dag:add_edge(prev, curr) + if prev_is_group and curr_is_group then + -- we use a fake task as a node to bridge the two groups. + local fakejob = {} + for _, job in ipairs(prev) do + if not dag:has_edge(job, fakejob) then + dag:add_edge(job, fakejob) + end + end + for _, job in ipairs(curr) do + if not dag:has_edge(fakejob, job) then + dag:add_edge(fakejob, job) + end + end + elseif curr_is_group then + for _, job in ipairs(curr) do + if not dag:has_edge(prev, job) then + dag:add_edge(prev, job) + end + end + elseif prev_is_group then + for _, job in ipairs(prev) do + if not dag:has_edge(job, curr) then + dag:add_edge(job, curr) + end + end + else + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) + end end end prev = curr + prev_is_group = curr_is_group end - -- TODO - -- add groups jobs end -- build a job queue -- cgit v1.3.1 From 5904d091ccdc6bde911d7ece763bd5253329ac9d Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 22 Mar 2025 22:32:49 +0800 Subject: update comment --- xmake/modules/async/jobgraph.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index edf3cfe81..cc4d0c743 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -119,7 +119,7 @@ function jobgraph:add_deps(...) assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then if prev_is_group and curr_is_group then - -- we use a fake task as a node to bridge the two groups. + -- we use a fake job as a node to bridge the two groups. local fakejob = {} for _, job in ipairs(prev) do if not dag:has_edge(job, fakejob) then -- cgit v1.3.1 From 502f7b6ddf39b8053c523d7a58532e0a5998c414 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 24 Mar 2025 22:49:41 +0800 Subject: add target_utils --- xmake/actions/build/build.lua | 42 +---------------- xmake/actions/build/build_files.lua | 45 +------------------ xmake/actions/build/prepare.lua | 87 ++++++++++++++++++++++++++++++++++++ xmake/actions/build/target_utils.lua | 73 ++++++++++++++++++++++++++++++ xmake/modules/async/jobgraph.lua | 5 +++ 5 files changed, 169 insertions(+), 83 deletions(-) create mode 100644 xmake/actions/build/target_utils.lua (limited to 'xmake/modules') diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index d11b1841b..e890faa0a 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -29,6 +29,7 @@ import("core.base.hashset") import("private.service.remote_cache.client", {alias = "remote_cache_client"}) import("private.service.distcc_build.client", {alias = "distcc_build_client"}) import("prepare", {alias = "prepare_build"}) +import("target_utils") -- clean target for rebuilding function _clean_target(target) @@ -234,46 +235,7 @@ end function get_batchjobs(targetnames, opt) -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end + local targets_root = target_utils.get_root_targets(targetnames, opt) -- generate batch jobs for default or all targets local jobrefs = {} diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 1b29cd6c0..ae09fc838 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -26,6 +26,7 @@ import("core.project.project") import("private.async.jobpool") import("async.runjobs") import("kinds.object") +import("target_utils") import("prepare_files", {alias = "prepare_build_files"}) -- match source files @@ -121,49 +122,7 @@ function _get_batchjobs(targetnames, opt) local filepatterns = _get_file_patterns(opt.sourcefiles) -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in pairs(project.targets()) do - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - for _, target in pairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - end + local targets_root = target_utils.get_root_targets(targetnames, opt) -- generate batch jobs for default or all targets local jobrefs = {} diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 52a0152af..45f0d75b9 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -21,6 +21,93 @@ -- imports import("core.base.option") import("core.project.config") +import("async.runjobs") +import("async.jobgraph", {alias = "async_jobgraph"}) + +-- get prepare jobs +function _get_prepare_jobs(targetnames, opt) + local jobgraph = async_jobgraph.new() + return jobgraph + + -- get root targets + --[[ + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + + -- generate batch jobs for default or all targets + local jobrefs = {} + local jobrefs_before = {} + local jobgraph = jobpool.new() + for _, target in ipairs(targets_root) do + _add_jobgraph_for_target_and_deps(jobgraph, jobgraph:rootjob(), target, jobrefs, jobrefs_before) + end + + -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 + for _, target in ipairs(project.ordertargets()) do + local target_job_before = jobrefs_before[target:name()] + if target_job_before then + for _, dep in ipairs(target:orderdeps()) do + if dep:policy("build.fence") then + local fence_job = jobrefs[dep:name()] + if fence_job then + jobgraph:add(fence_job, target_job_before) + end + end + end + end + end + + return jobgraph]] +end function main(targetnames, opt) + local jobgraph = _get_prepare_jobs(targetnames, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + runjobs("prepare", jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir}) + os.cd(curdir) + end end diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua new file mode 100644 index 000000000..2c098400d --- /dev/null +++ b/xmake/actions/build/target_utils.lua @@ -0,0 +1,73 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file target_utils.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.config") +import("core.project.project") + +-- get all root targets +function get_root_targets(targetnames, opt) + opt = opt or {} + + -- get root targets + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + return targets_root +end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index cc4d0c743..3767cfe73 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -176,6 +176,11 @@ function jobgraph:size() return self._size end +-- is empty? +function jobgraph:empty() + return self:size() == 0 +end + -- tostring function jobgraph:__tostring() return string.format("", self:name() or "anonymous", self:size()) -- cgit v1.3.1 From 0a01105ef8e83039523a4aa97cb3ca0c8726df04 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 24 Mar 2025 22:50:28 +0800 Subject: pass target roots as args --- xmake/actions/build/build.lua | 14 ++-- xmake/actions/build/build_files.lua | 12 ++-- xmake/actions/build/prepare.lua | 47 +------------- xmake/actions/build/prepare_files.lua | 2 +- xmake/modules/private/diagnosis/dump_buildjobs.lua | 30 --------- xmake/modules/private/diagnosis/dump_targets.lua | 74 ---------------------- 6 files changed, 17 insertions(+), 162 deletions(-) delete mode 100644 xmake/modules/private/diagnosis/dump_buildjobs.lua delete mode 100644 xmake/modules/private/diagnosis/dump_targets.lua (limited to 'xmake/modules') diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index e890faa0a..05284490d 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -231,11 +231,8 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, end end --- get batch jobs, @note we need to export it for private.diagnosis.dump_buildjobs -function get_batchjobs(targetnames, opt) - - -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) +-- get batch jobs +function _get_batchjobs(targets_root, opt) -- generate batch jobs for default or all targets local jobrefs = {} @@ -265,8 +262,11 @@ end function main(targetnames, opt) + -- get root targets + local targets_root = target_utils.get_root_targets(targetnames, opt) + -- prepare to build - prepare_build(targetnames, opt) + prepare_build(targets_root, opt) -- enable distcc? local distcc @@ -275,7 +275,7 @@ function main(targetnames, opt) end -- build all jobs - local batchjobs = get_batchjobs(targetnames, opt) + local batchjobs = _get_batchjobs(targets_root, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build", batchjobs, {on_exit = function (errors) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index ae09fc838..8d1383cbb 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -116,14 +116,11 @@ function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, end -- get batch jobs -function _get_batchjobs(targetnames, opt) +function _get_batchjobs(targets_root, opt) -- convert all sourcefiles to lua pattern local filepatterns = _get_file_patterns(opt.sourcefiles) - -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) - -- generate batch jobs for default or all targets local jobrefs = {} local batchjobs = jobpool.new() @@ -180,11 +177,14 @@ end -- the main entry function main(targetnames, opt) + -- get root targets + local targets_root = target_utils.get_root_targets(targetnames, opt) + -- prepare to build files - prepare_build_files(targetnames, opt) + prepare_build_files(targets_root, opt) -- build all jobs - local batchjobs = _get_batchjobs(targetnames, opt) + local batchjobs = _get_batchjobs(targets_root, opt) if batchjobs and batchjobs:size() > 0 then local curdir = os.curdir() runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) diff --git a/xmake/actions/build/prepare.lua b/xmake/actions/build/prepare.lua index 45f0d75b9..aea951168 100644 --- a/xmake/actions/build/prepare.lua +++ b/xmake/actions/build/prepare.lua @@ -25,52 +25,11 @@ import("async.runjobs") import("async.jobgraph", {alias = "async_jobgraph"}) -- get prepare jobs -function _get_prepare_jobs(targetnames, opt) +function _get_prepare_jobs(targets_root, opt) local jobgraph = async_jobgraph.new() return jobgraph - -- get root targets --[[ - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end -- generate batch jobs for default or all targets local jobrefs = {} @@ -98,8 +57,8 @@ function _get_prepare_jobs(targetnames, opt) return jobgraph]] end -function main(targetnames, opt) - local jobgraph = _get_prepare_jobs(targetnames, opt) +function main(targets_root, opt) + local jobgraph = _get_prepare_jobs(targets_root, opt) if jobgraph and not jobgraph:empty() then local curdir = os.curdir() runjobs("prepare", jobgraph, {on_exit = function (errors) diff --git a/xmake/actions/build/prepare_files.lua b/xmake/actions/build/prepare_files.lua index acdc41b0e..3abb3ef31 100644 --- a/xmake/actions/build/prepare_files.lua +++ b/xmake/actions/build/prepare_files.lua @@ -22,5 +22,5 @@ import("core.base.option") import("core.project.config") -function main(targetnames, opt) +function main(targets_root, opt) end diff --git a/xmake/modules/private/diagnosis/dump_buildjobs.lua b/xmake/modules/private/diagnosis/dump_buildjobs.lua deleted file mode 100644 index 16083a40b..000000000 --- a/xmake/modules/private/diagnosis/dump_buildjobs.lua +++ /dev/null @@ -1,30 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file dump_buildjobs.lua --- - --- imports -import("core.project.config") -import("actions.build.build", {rootdir = os.programdir()}) - --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - print(build.get_batchjobs(targetname)) -end - diff --git a/xmake/modules/private/diagnosis/dump_targets.lua b/xmake/modules/private/diagnosis/dump_targets.lua deleted file mode 100644 index c992a28f0..000000000 --- a/xmake/modules/private/diagnosis/dump_targets.lua +++ /dev/null @@ -1,74 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file dump_targets.lua --- - --- imports -import("core.base.hashset") -import("core.project.config") -import("core.project.project") - --- get targets -function _get_targets(targetname) - - -- get targets - local targets = {} - if targetname then - table.insert(targets, project.target(targetname)) - else - for _, target in pairs(project.targets()) do - table.insert(targets, target) - end - end - return targets -end - --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - for _, target in ipairs(_get_targets(targetname)) do - cprint("${bright}target(%s):${clear} %s", target:name(), target:kind()) - local deps = target:get("deps") - if deps then - cprint(" ${color.dump.string}deps:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(deps), ", ")) - end - local options = {} - for _, optname in ipairs(target:get("options")) do - if not optname:startswith("__") then - table.insert(options, optname) - end - end - if #options > 0 then - cprint(" ${color.dump.string}options:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(options), ", ")) - end - local packages = target:get("packages") - if packages then - cprint(" ${color.dump.string}packages:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(packages), ", ")) - end - local rules = target:get("rules") - if rules then - cprint(" ${color.dump.string}rules:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(rules), ", ")) - end - print("") - end -end - -- cgit v1.3.1 From 4fee5fe56d3eedb3871ee237b083bb3bcbf6ec5c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:41:57 +0800 Subject: fix jobgraph for isolated vertex --- xmake/actions/build/target_utils.lua | 4 ++-- xmake/core/base/graph.lua | 11 +++++++++++ xmake/modules/async/jobgraph.lua | 4 +++- 3 files changed, 16 insertions(+), 3 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 6a29f2e78..74c57a2c9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -37,7 +37,7 @@ function _add_jobs_for_target(jobgraph, target, opt) -- add after_xxx jobs for target local job_kind = opt.job_kind local job_after = target:fullname() .. "/after_" .. job_kind - --[[ + -- TODO we should remove root job, use group instead of it jobgraph:add(job_after, function (index, total, opt) local progress = opt.progress local script_aftername = job_kind .. "_after" @@ -59,7 +59,7 @@ function _add_jobs_for_target(jobgraph, target, opt) end end end - end)]] + end) end -- add jobs for the given target and deps diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 658b71e73..43b43ce3c 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -94,6 +94,17 @@ function graph:has_vertex(v) return table.contains(self:vertices(), v) end +-- add an isolated without edges +function graph:add_vertex(v) + if not self:has_vertex(v) then + table.insert(self._vertices, v) + self._adjacent_edges[v] = {} + end + + -- reset partial topological sort state since graph structure changed + self._partial_topo_dirty = true +end + -- remove the given vertex? function graph:remove_vertex(v) local contains = false diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3767cfe73..d737c7348 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -70,10 +70,12 @@ end -- function jobgraph:add(name, run, opt) opt = opt or {} + local dag = self._dag local jobs = self._jobs if not jobs[name] then local job = {name = name, run = run, opt = opt} jobs[name] = job + dag:add_vertex(job) self._size = self._size + 1 if opt.groups then @@ -91,9 +93,9 @@ end -- remove a given job function jobgraph:remove(name) + local dag = self._dag local jobs = self._jobs local job = jobs[name] - local dag = self._dag if job then assert(self._size > 0) jobs[name] = nil -- cgit v1.3.1 From 78db72c72e2f2d1c699c7c7010e6bd84cdbf9644 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:48:17 +0800 Subject: add stage jobs for target --- xmake/actions/build/target_utils.lua | 43 ++++++++++++++++---- xmake/modules/async/jobgraph.lua | 77 ++++++++++++++++++++---------------- 2 files changed, 77 insertions(+), 43 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 74c57a2c9..aa6f06aa9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,17 +27,30 @@ import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") --- add jobs for the given target -function _add_jobs_for_target(jobgraph, target, opt) +-- add stage jobs for the given target +-- stage: before, after or "" +function _add_stage_jobs_for_target(jobgraph, target, stage, opt) opt = opt or {} - if not target:is_enabled() then - return + local job_kind = opt.job_kind + + -- the stage group, e.g. foo/after_prepare, bar/before_build + local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) + + -- call target script first, e.g. before/after_prepare, before/after_build + local progress = opt.progress + local script_name = job_kind .. "_" .. stage + local script = target:script(script_name) + if script then + local jobname = target:fullname() .. "/" .. script_name + jobgraph:add(jobname, function (index, total, opt) + -- TODO bind target envs + script(target, {progress = progress}) + end, {groups = group_name}) end - -- add after_xxx jobs for target - local job_kind = opt.job_kind - local job_after = target:fullname() .. "/after_" .. job_kind + -- TODO we should remove root job, use group instead of it + --[[ jobgraph:add(job_after, function (index, total, opt) local progress = opt.progress local script_aftername = job_kind .. "_after" @@ -59,7 +72,21 @@ function _add_jobs_for_target(jobgraph, target, opt) end end end - end) + end)]] +end + +-- add jobs for the given target +function _add_jobs_for_target(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + -- add group jobs for target, e.g. after_xxx -> (depend on) on_xxx -> before_xxx + local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) + local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) + local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) + jobgraph:add_deps(group_after, group, group_before) end -- add jobs for the given target and deps diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index d737c7348..3440f606b 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -105,54 +105,61 @@ function jobgraph:remove(name) end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +-- +-- and it supports nil, e.g add_deps("foo", nil, "bar", ...) +-- function jobgraph:add_deps(...) local prev local prev_is_group local dag = self._dag local jobs = self._jobs local groups = self._groups - for _, name in ipairs(table.pack(...)) do - local curr_is_group = false - local curr = jobs[name] - if not curr then - curr = groups[name] - curr_is_group = true - end - assert(curr, "job(%s) not found in jobgraph(%s)", name, self) - if prev then - if prev_is_group and curr_is_group then - -- we use a fake job as a node to bridge the two groups. - local fakejob = {} - for _, job in ipairs(prev) do - if not dag:has_edge(job, fakejob) then - dag:add_edge(job, fakejob) + local deps = table.pack(...) + for i = 1, deps.n do + local name = deps[i] + if name then + local curr_is_group = false + local curr = jobs[name] + if not curr then + curr = groups[name] + curr_is_group = true + end + assert(curr, "job(%s) not found in jobgraph(%s)", name, self) + if prev then + if prev_is_group and curr_is_group then + -- we use a fake job as a node to bridge the two groups. + local fakejob = {} + for _, job in ipairs(prev) do + if not dag:has_edge(job, fakejob) then + dag:add_edge(job, fakejob) + end end - end - for _, job in ipairs(curr) do - if not dag:has_edge(fakejob, job) then - dag:add_edge(fakejob, job) + for _, job in ipairs(curr) do + if not dag:has_edge(fakejob, job) then + dag:add_edge(fakejob, job) + end end - end - elseif curr_is_group then - for _, job in ipairs(curr) do - if not dag:has_edge(prev, job) then - dag:add_edge(prev, job) + elseif curr_is_group then + for _, job in ipairs(curr) do + if not dag:has_edge(prev, job) then + dag:add_edge(prev, job) + end end - end - elseif prev_is_group then - for _, job in ipairs(prev) do - if not dag:has_edge(job, curr) then - dag:add_edge(job, curr) + elseif prev_is_group then + for _, job in ipairs(prev) do + if not dag:has_edge(job, curr) then + dag:add_edge(job, curr) + end + end + else + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) end - end - else - if not dag:has_edge(prev, curr) then - dag:add_edge(prev, curr) end end + prev = curr + prev_is_group = curr_is_group end - prev = curr - prev_is_group = curr_is_group end end -- cgit v1.3.1 From 37c593a71339240ea880ab5478f2f5120af992cc Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 00:54:11 +0800 Subject: improve jobs --- xmake/actions/build/target_utils.lua | 62 +++++++++++++++++------------------- xmake/modules/async/jobgraph.lua | 8 ++++- 2 files changed, 37 insertions(+), 33 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index aa6f06aa9..aa5413995 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -32,47 +32,45 @@ import("private.utils.batchcmds") function _add_stage_jobs_for_target(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind + local progress = opt.progress - -- the stage group, e.g. foo/after_prepare, bar/before_build + -- the group name, e.g. foo/after_prepare, bar/before_build local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) - -- call target script first, e.g. before/after_prepare, before/after_build - local progress = opt.progress - local script_name = job_kind .. "_" .. stage - local script = target:script(script_name) - if script then - local jobname = target:fullname() .. "/" .. script_name - jobgraph:add(jobname, function (index, total, opt) - -- TODO bind target envs - script(target, {progress = progress}) - end, {groups = group_name}) - end + -- the script name, e.g. before/after_prepare, before/after_build + local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind + -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd + local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - -- TODO we should remove root job, use group instead of it - --[[ - jobgraph:add(job_after, function (index, total, opt) - local progress = opt.progress - local script_aftername = job_kind .. "_after" - local script_after = target:script(script_aftername) - if script_after then - script_after(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local script_after = r:script(script_aftername) - if script_after then - script_after(target, {progress = progress}) - else - local scriptcmd_aftername = job_kind .. "cmd_after" - local scriptcmd_after = r:script(scriptcmd_aftername) - if scriptcmd_after then + -- call target and rules script + local jobdeps = {} + local instances = table.join(target, target:orderules()) -- TODO sort them + for _, instance in ipairs(instances) do + local script = instance:script(script_name) + if script then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) + jobgraph:add(jobname, function (index, total, opt) + -- TODO bind target envs + script(target, {progress = progress}) + end, {groups = group_name}) + table.insert(jobdeps, jobname) + else + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_after(target, batchcmds_, {progress = progress}) + scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end + end, {groups = group_name}) + table.insert(jobdeps, jobname) end end - end)]] + end + + -- add job deps + jobgraph:add_deps(jobdeps) end -- add jobs for the given target diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 3440f606b..b72fb5b16 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -107,6 +107,7 @@ end -- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... -- -- and it supports nil, e.g add_deps("foo", nil, "bar", ...) +-- and it also supports to add deps list, e.g. add_deps(deps) -- function jobgraph:add_deps(...) local prev @@ -115,7 +116,12 @@ function jobgraph:add_deps(...) local jobs = self._jobs local groups = self._groups local deps = table.pack(...) - for i = 1, deps.n do + local count = deps.n + if count == 1 and type(deps[1]) == "table" then + deps = deps[1] + count = #deps + end + for i = 1, count do local name = deps[i] if name then local curr_is_group = false -- cgit v1.3.1 From 8beb0732ff716cd5b3d7a40744b1d239302fdb4b Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 22:49:41 +0800 Subject: rename add_deps to add_orders --- tests/modules/async/run_jobgraph.lua | 4 ++-- xmake/actions/build/target_utils.lua | 15 +++++++++------ xmake/core/base/graph.lua | 18 +++++++++++------- xmake/modules/async/jobgraph.lua | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 25 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index bc1b5500f..9b8f529ed 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -18,7 +18,7 @@ function _test_basic() jobs:add("job/" .. i, _jobfunc) for j = 1, 50 do jobs:add("job/" .. i .. "/" .. j, _jobfunc) - jobs:add_deps("job/" .. i .. "/" .. j, "job/" .. i, "job/root") + jobs:add_orders("job/" .. i .. "/" .. j, "job/" .. i, "job/root") end end t = os.mclock() @@ -37,7 +37,7 @@ function _test_group() jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) end end - jobs:add_deps("foo", "bar", "job/root") + jobs:add_orders("foo", "bar", "job/root") t = os.mclock() runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 1fef599ac..b91b436f0 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -50,7 +50,7 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) end -- call target and rules script - local jobdeps = {} + local joborders = {} for _, instance in ipairs(instances) do local script = instance:script(script_name) if script then @@ -59,7 +59,7 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- TODO bind target envs script(target, {progress = progress}) end, {groups = group_name}) - table.insert(jobdeps, jobname) + table.insert(joborders, jobname) else local scriptcmd = instance:script(scriptcmd_name) if scriptcmd then @@ -70,13 +70,16 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) scriptcmd(target, batchcmds_, {progress = progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end, {groups = group_name}) - table.insert(jobdeps, jobname) + table.insert(joborders, jobname) end end end - -- add job deps - jobgraph:add_deps(jobdeps) + -- add job orders + if #joborders > 0 then + jobgraph:add_orders(joborders) + return group_name + end end -- add jobs for the given target @@ -90,7 +93,7 @@ function _add_jobs_for_target(jobgraph, target, opt) local group = _add_stage_jobs_for_target(jobgraph, target, "", opt) local group_before = _add_stage_jobs_for_target(jobgraph, target, "before", opt) local group_after = _add_stage_jobs_for_target(jobgraph, target, "after", opt) - jobgraph:add_deps(group_after, group, group_before) + jobgraph:add_orders(group_before, group, group_after) end -- add jobs for the given target and deps diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 43b43ce3c..db8682b21 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -173,12 +173,14 @@ end -- -- e.g. -- --- add_edge(a, b) -- a depend on b --- add_edge(b, c) -- b depend on c +-- edges: a (indegree: 0) -> b -> c -- --- local node1, has_cycle = g:partial_topo_sort_next() -- return c +-- add_edge(a, b) +-- add_edge(b, c) +-- +-- local node1, has_cycle = g:partial_topo_sort_next() -- return a -- local node2, has_cycle = g:partial_topo_sort_next() -- return b --- local node3, has_cycle = g:partial_topo_sort_next() -- return a +-- local node3, has_cycle = g:partial_topo_sort_next() -- return c -- local node4, has_cycle = g:partial_topo_sort_next() -- return nil (empty, all done) -- function graph:partial_topo_sort_next() @@ -250,10 +252,12 @@ end -- -- e.g. -- --- add_edge(a, b) -- a depend on b --- add_edge(b, c) -- b depend on c +-- edges: a (indegree: 0) -> b -> c +-- +-- add_edge(a, b) +-- add_edge(b, c) -- --- it will return {c, b, a} +-- it will return {a, b, c} function graph:topo_sort() if not self:is_directed() then return diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index b72fb5b16..fe42f6647 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -104,25 +104,25 @@ function jobgraph:remove(name) end end --- add job deps, e.g. add_deps(a, b, c, ...): a -> b -> c, ... +-- add job orders, e.g. add_orders(a, b, c, ...): a -> b -> c, ... -- --- and it supports nil, e.g add_deps("foo", nil, "bar", ...) --- and it also supports to add deps list, e.g. add_deps(deps) +-- and it supports nil, e.g add_orders("foo", nil, "bar", ...) +-- and it also supports to add orders list, e.g. add_orders(orders) -- -function jobgraph:add_deps(...) +function jobgraph:add_orders(...) local prev local prev_is_group local dag = self._dag local jobs = self._jobs local groups = self._groups - local deps = table.pack(...) - local count = deps.n - if count == 1 and type(deps[1]) == "table" then - deps = deps[1] - count = #deps + local orders = table.pack(...) + local count = orders.n + if count == 1 and type(orders[1]) == "table" then + orders = orders[1] + count = #orders end for i = 1, count do - local name = deps[i] + local name = orders[i] if name then local curr_is_group = false local curr = jobs[name] -- cgit v1.3.1 From f55ca39e18390317210ac7d660cbfaa517c1e691 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 27 Mar 2025 23:05:03 +0800 Subject: add jobgraph:group --- tests/modules/async/run_jobgraph.lua | 8 +++++--- xmake/actions/build/target_utils.lua | 21 ++++++++++++--------- xmake/modules/async/jobgraph.lua | 27 +++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 14 deletions(-) (limited to 'xmake/modules') diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua index 9b8f529ed..4463038b7 100644 --- a/tests/modules/async/run_jobgraph.lua +++ b/tests/modules/async/run_jobgraph.lua @@ -33,9 +33,11 @@ function _test_group() jobs:add("job/root", _jobfunc) for i = 1, 3 do jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) - for j = 1, 50 do - jobs:add("job/" .. i .. "/" .. j, _jobfunc, {groups = "foo"}) - end + jobgraph:group("foo", function () + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + end + end) end jobs:add_orders("foo", "bar", "job/root") t = os.mclock() diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 41c979b85..13cf54ad9 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -39,7 +39,6 @@ end function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) opt = opt or {} local joborders = opt.joborders - local group_name = opt.group_name local script = instance:script(script_name) if script then -- call custom script with jobgraph @@ -49,6 +48,7 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) -- on_build(function (target, jobgraph, opt) -- end, {jobgraph = true}) if instance:extraconf(script_name, "jobgraph") then + -- TODO group and joborders script(target, jobgraph) elseif instance:extraconf(script_name, "batch") then wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it.", instance:fullname(), script_name) @@ -62,9 +62,11 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) local jobname = string.format("%s/%s/%s", instance == target and "target" or "rule", instance:fullname(), script_name) jobgraph:add(jobname, function (index, total, opt) script(target, {progress = opt.progress}) - end, {groups = group_name}) + end) table.insert(joborders, jobname) end + elseif false then + -- TODO call builtin script else -- call command script -- e.g. @@ -79,7 +81,7 @@ function _add_script_job(jobgraph, instance, script_name, scriptcmd_name, opt) local batchcmds_ = batchcmds.new({target = target}) scriptcmd(target, batchcmds_, {progress = opt.progress}) batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {groups = group_name}) + end) table.insert(joborders, jobname) end end @@ -108,12 +110,13 @@ function _add_stage_jobs_for_target(jobgraph, target, stage, opt) -- call target and rules script local joborders = {} - for _, instance in ipairs(instances) do - _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { - group_name = group_name, - joborders = joborders - }) - end + jobgraph:group(group_name, function () + for _, instance in ipairs(instances) do + _add_script_job(jobgraph, instance, script_name, scriptcmd_name, { + joborders = joborders + }) + end + end) -- add job orders if #joborders > 0 then diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index fe42f6647..b9e9a46da 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -78,8 +78,9 @@ function jobgraph:add(name, run, opt) dag:add_vertex(job) self._size = self._size + 1 - if opt.groups then - for _, group_name in ipairs(opt.groups) do + if self._current_groups or opt.groups then + local job_groups = table.join(self._current_groups or {}, opt.groups) + for _, group_name in ipairs(job_groups) do local groups = self._groups[group_name] if not groups then groups = {} @@ -104,6 +105,28 @@ function jobgraph:remove(name) end end +-- enter group to add jobs +-- +-- e.g. +-- jobgraph:group("foo", function () +-- jobgraph:add("job1", function (index, total, opt) +-- TODO +-- end) +-- jobgraph:add("job2", function (index, total, opt) +-- TODO +-- end) +-- end) +function jobgraph:group(name, callback) + local current_groups = self._current_groups + if current_groups == nil then + current_groups = {} + self._current_groups = current_groups + end + table.insert(current_groups, name) + callback() + table.remove(current_groups) +end + -- add job orders, e.g. add_orders(a, b, c, ...): a -> b -> c, ... -- -- and it supports nil, e.g add_orders("foo", nil, "bar", ...) -- cgit v1.3.1 From 4b47fa99489e59337b056e0c95b50493c050fe32 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:24:46 +0800 Subject: improve to build object to support jobgraph --- xmake/modules/private/action/build/object.lua | 27 +++++++++++++++++++++++++-- xmake/rules/c++/xmake.lua | 4 ++-- 2 files changed, 27 insertions(+), 4 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index ae6bf9f4a..8f71f4b68 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -143,8 +143,8 @@ function build(target, sourcebatch, opt) end end --- add batch jobs to build the source files -function main(target, batchjobs, sourcebatch, opt) +-- add build jobs to batchjobs +function _add_batchjobs(target, batchjobs, sourcebatch, opt) local rootjob = opt.rootjob for i = 1, #sourcebatch.sourcefiles do local sourcefile = sourcebatch.sourcefiles[i] @@ -157,3 +157,26 @@ function main(target, batchjobs, sourcebatch, opt) end, {rootjob = rootjob, distcc = opt.distcc}) end end + +-- add build jobs to jobgraph +function _add_jobgraph(target, jobgraph, sourcebatch, opt) + for i = 1, #sourcebatch.sourcefiles do + local sourcefile = sourcebatch.sourcefiles[i] + local objectfile = sourcebatch.objectfiles[i] + local dependfile = sourcebatch.dependfiles[i] + local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) + local jobname = target:fullname() .. "/" .. sourcefile + jobgraph:add(jobname, function (index, total, jobopt) + local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) + build_object(target, sourcefile, build_opt) + end, {distcc = opt.distcc}) + end +end + +function main(target, jobs, sourcebatch, opt) + if jobs.add_orders then + _add_jobgraph(target, jobs, sourcebatch, opt) + else + _add_batchjobs(target, jobs, sourcebatch, opt) + end +end diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua index 94bba431e..abe6cf334 100644 --- a/xmake/rules/c++/xmake.lua +++ b/xmake/rules/c++/xmake.lua @@ -21,7 +21,7 @@ rule("c.build") set_sourcekinds("cc") add_deps("c.build.pcheader", "c.build.optimization", "c.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) -- enable vs runtime as MD by default if target:is_plat("windows") and not target:get("runtimes") then @@ -43,7 +43,7 @@ rule("c.build") rule("c++.build") set_sourcekinds("cxx") add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization", "c++.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) -- enable c++ exceptions by default if target:is_plat("windows") and not target:get("exceptions") then -- cgit v1.3.1 From 92d6126d1c98c9c58d38f5b173d7248a1e882d20 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 29 Mar 2025 22:32:11 +0800 Subject: enable more build.object to jobgraph --- xmake/modules/private/action/build/object.lua | 8 ++++---- xmake/rules/asm/xmake.lua | 2 +- xmake/rules/cuda/xmake.lua | 2 +- xmake/rules/dlang/xmake.lua | 2 +- xmake/rules/objc++/xmake.lua | 4 ++-- xmake/rules/swift/xmake.lua | 2 +- xmake/rules/winsdk/xmake.lua | 2 +- xmake/rules/zig/xmake.lua | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 8f71f4b68..5118dbb82 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -173,10 +173,10 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) end end -function main(target, jobs, sourcebatch, opt) - if jobs.add_orders then - _add_jobgraph(target, jobs, sourcebatch, opt) +function main(target, jobgraph, sourcebatch, opt) + if jobgraph.add_orders then + _add_jobgraph(target, jobgraph, sourcebatch, opt) else - _add_batchjobs(target, jobs, sourcebatch, opt) + _add_batchjobs(target, jobgraph, sourcebatch, opt) end end diff --git a/xmake/rules/asm/xmake.lua b/xmake/rules/asm/xmake.lua index 78b693137..1c6743793 100644 --- a/xmake/rules/asm/xmake.lua +++ b/xmake/rules/asm/xmake.lua @@ -21,7 +21,7 @@ -- define rule: asm.build rule("asm.build") set_sourcekinds("as") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: asm rule("asm") diff --git a/xmake/rules/cuda/xmake.lua b/xmake/rules/cuda/xmake.lua index 334f569f1..4ce6699ff 100644 --- a/xmake/rules/cuda/xmake.lua +++ b/xmake/rules/cuda/xmake.lua @@ -22,7 +22,7 @@ rule("cuda.build") set_sourcekinds("cu") add_deps("cuda.build.devlink") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- https://github.com/xmake-io/xmake/issues/4755 local cu_ccbin = target:tool("cu-ccbin") diff --git a/xmake/rules/dlang/xmake.lua b/xmake/rules/dlang/xmake.lua index 7ffb8c42f..f5e741f7a 100644 --- a/xmake/rules/dlang/xmake.lua +++ b/xmake/rules/dlang/xmake.lua @@ -21,7 +21,7 @@ rule("dlang.build") set_sourcekinds("dc") add_deps("dlang.build.optimization") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_load(function (target) local toolchains = target:get("toolchains") or get_config("toolchain") if not toolchains or not table.contains(table.wrap(toolchains), "dlang", "dmd", "ldc", "gdc") then diff --git a/xmake/rules/objc++/xmake.lua b/xmake/rules/objc++/xmake.lua index c653261e7..3714ddf3f 100644 --- a/xmake/rules/objc++/xmake.lua +++ b/xmake/rules/objc++/xmake.lua @@ -31,7 +31,7 @@ rule("objc.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc++.build rule("objc++.build") @@ -46,7 +46,7 @@ rule("objc++.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc rule("objc++") diff --git a/xmake/rules/swift/xmake.lua b/xmake/rules/swift/xmake.lua index d1ea0e886..9a2dfbb5b 100644 --- a/xmake/rules/swift/xmake.lua +++ b/xmake/rules/swift/xmake.lua @@ -21,7 +21,7 @@ -- define rule: swift.build rule("swift.build") set_sourcekinds("sc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- we use swift-frontend to support multiple modules -- @see https://github.com/xmake-io/xmake/issues/3916 diff --git a/xmake/rules/winsdk/xmake.lua b/xmake/rules/winsdk/xmake.lua index 0544abb86..6461c4eca 100644 --- a/xmake/rules/winsdk/xmake.lua +++ b/xmake/rules/winsdk/xmake.lua @@ -21,7 +21,7 @@ -- define rule: win.sdk.resource rule("win.sdk.resource") set_sourcekinds("mrc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: application rule("win.sdk.application") diff --git a/xmake/rules/zig/xmake.lua b/xmake/rules/zig/xmake.lua index 4565fcda6..3d0a0a4b9 100644 --- a/xmake/rules/zig/xmake.lua +++ b/xmake/rules/zig/xmake.lua @@ -26,7 +26,7 @@ rule("zig.build") os.mkdir(cachedir) target:add("zcflags", "--cache-dir " .. cachedir) end) - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: zig rule("zig") -- cgit v1.3.1 From c0867ffd08915f2f80decbf97da17b685fb74525 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 23:05:27 +0800 Subject: fix build modules --- xmake/modules/private/action/build/object.lua | 2 +- .../rules/c++/modules/modules_support/builder.lua | 26 +++++++++++----------- .../c++/modules/modules_support/clang/builder.lua | 6 ++--- .../c++/modules/modules_support/gcc/builder.lua | 7 +++--- .../c++/modules/modules_support/msvc/builder.lua | 7 +++--- xmake/rules/c++/modules/xmake.lua | 11 +++------ 6 files changed, 28 insertions(+), 31 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 5118dbb82..0028b4b6f 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -165,7 +165,7 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) local objectfile = sourcebatch.objectfiles[i] local dependfile = sourcebatch.dependfiles[i] local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) - local jobname = target:fullname() .. "/" .. sourcefile + local jobname = target:fullname() .. "/obj/" .. sourcefile jobgraph:add(jobname, function (index, total, jobopt) local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) build_object(target, sourcefile, build_opt) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 59f00b64f..0fd9920b1 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -46,7 +46,7 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep table.insert(deps, depname) end @@ -263,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -279,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/" .. (name or cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -292,11 +292,11 @@ end -- build modules for jobgraph function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) local jobdeps = {} - local build_modules_group = target:fullname() .. "/build_modules" + local build_modules_group = target:fullname() .. "/module/build_modules" jobgraph:group(build_modules_group, function () -- add populate module job - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" jobgraph:add(populate_jobname, function(index, total, opt) _try_reuse_modules(target, modules) _builder(target).populate_module_map(target, modules) @@ -305,7 +305,7 @@ function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local jobname = target:fullname() .. "/" .. (name or cppfile) + local jobname = target:fullname() .. "/module/" .. (name or cppfile) _builder(target).make_module_jobgraph(target, jobgraph, { module = module, objectfile = objectfile, cppfile = cppfile }) @@ -349,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -385,14 +385,14 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local build_modules_group = target:fullname() .. "/build_modules" - local build_headerunits_group = target:fullname() .. "/build_headerunits" + local build_modules_group = target:fullname() .. "/module/build_modules" + local build_headerunits_group = target:fullname() .. "/module/build_headerunits" jobgraph:group(build_headerunits_group, function () local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key _builder(target).make_headerunit_buildjobs(target, job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) end @@ -458,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index dd9a6fad5..ec6692c35 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -285,7 +285,7 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 38a9e3d04..05e8380da 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -247,7 +247,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index fe1f25bb4..8f582438b 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -263,9 +263,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then @@ -331,7 +331,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f4b7f744e..42eab0c19 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -112,15 +112,12 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles + -- build modules and headerunits, and we need to build headerunits first if jobgraph.add_orders then - -- build modules builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - else - -- build modules, deprecated + else -- deprecated builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) end @@ -179,10 +176,8 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build headerunits + -- build headerunits and modules builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - - -- build modules builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles -- cgit v1.3.1 From e52e6ad4b4ee9cd20aa49a3da87a2c20e88f65f1 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 1 Apr 2025 00:55:39 +0800 Subject: check to add job --- xmake/modules/async/jobgraph.lua | 2 ++ 1 file changed, 2 insertions(+) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index b9e9a46da..28126757f 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -89,6 +89,8 @@ function jobgraph:add(name, run, opt) table.insert(groups, job) end end + else + raise("job(%s): has already been added!", name) end end -- cgit v1.3.1 From 38695f0bf5875dc5db9ccaa8c9ddbf8ef46d4f5e Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 3 Apr 2025 00:47:55 +0800 Subject: add target deps orders --- xmake/actions/build/target_utils.lua | 7 +++++++ xmake/modules/async/jobgraph.lua | 5 +++++ 2 files changed, 12 insertions(+) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3edf9f42b..9529fd490 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -287,9 +287,16 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) if not targetrefs[targetname] then targetrefs[targetname] = target add_targetjobs(jobgraph, target, opt) + + local linkjob = target:fullname() .. "/link_objects" for _, depname in ipairs(target:get("deps")) do local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + + local linkjob_dep = dep:fullname() .. "/link_objects" + if jobgraph:has(linkjob) and jobgraph:has(linkjob_dep) then + jobgraph:add_orders(linkjob_dep, linkjob) + end end end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 28126757f..612ff4882 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -107,6 +107,11 @@ function jobgraph:remove(name) end end +-- has the given job or group? +function jobgraph:has(name) + return self._jobs[name] or self._groups[name] +end + -- enter group to add jobs -- -- e.g. -- cgit v1.3.1 From 15d9e01ab37a28d861ce58b1b3701779e94da865 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 4 Apr 2025 00:42:58 +0800 Subject: dump jobgraph --- xmake/actions/build/target_utils.lua | 4 ++-- xmake/core/base/graph.lua | 13 ++++++------ xmake/modules/async/jobgraph.lua | 38 ++++++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 14 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 3d8b5e361..85da7816e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -181,7 +181,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) local job_kind = opt.job_kind -- the group name, e.g. foo/after_prepare, bar/before_build - local group_name = string.format("%s/%s_%s", target:fullname(), stage, job_kind) + local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) -- the script name, e.g. before/after_prepare, before/after_build local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind @@ -465,7 +465,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) local job_kindcmd_files = job_kind .. "cmd_files" -- the group name, e.g. foo/after_prepare_files, bar/before_build_files - local group_name = string.format("%s/%s_%s_files", target:fullname(), stage, job_kind) + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) -- the script name, e.g. before/after_prepare_files, before/after_build_files local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 53f09c59f..fd7675628 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -23,6 +23,7 @@ local table = require("base/table") local queue = require("base/queue") local object = require("base/object") local hashset = require("base/hashset") +local utils = require("base/utils") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -454,15 +455,15 @@ end function graph:dump() local vertices = self:vertices() local edges = self:edges() - print(string.format("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges)) - print("vertices: ") + utils.cprint("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges) + utils.cprint("vertices: ") for _, v in ipairs(vertices) do - print(string.format(" %s", v)) + utils.cprint(" %s", v) end - print("") - print("edges: ") + utils.cprint("") + utils.cprint("edges: ") for _, e in ipairs(edges) do - print(string.format(" %s -> %s", e:from(), e:to())) + utils.cprint(" %s ${color.dump.reference}->${clear} %s", e:from(), e:to()) end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 612ff4882..be1ad57f0 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -142,6 +142,7 @@ end function jobgraph:add_orders(...) local prev local prev_is_group + local prev_name local dag = self._dag local jobs = self._jobs local groups = self._groups @@ -163,16 +164,16 @@ function jobgraph:add_orders(...) assert(curr, "job(%s) not found in jobgraph(%s)", name, self) if prev then if prev_is_group and curr_is_group then - -- we use a fake job as a node to bridge the two groups. - local fakejob = {} + -- we use a bridge job as a node to bridge the two groups. + local bridge = {from_group = prev_name, to_group = name} for _, job in ipairs(prev) do - if not dag:has_edge(job, fakejob) then - dag:add_edge(job, fakejob) + if not dag:has_edge(job, bridge) then + dag:add_edge(job, bridge) end end for _, job in ipairs(curr) do - if not dag:has_edge(fakejob, job) then - dag:add_edge(fakejob, job) + if not dag:has_edge(bridge, job) then + dag:add_edge(bridge, job) end end elseif curr_is_group then @@ -195,6 +196,7 @@ function jobgraph:add_orders(...) end prev = curr prev_is_group = curr_is_group + prev_name = name end end end @@ -226,6 +228,30 @@ function jobgraph:empty() return self:size() == 0 end +-- dump jobgraph +function jobgraph:dump() + print("================================ %s ================================", self) + for _, node in ipairs(self._dag:vertices()) do + debug.setmetatable(node, {__tostring = function (v) + if v.from_group and v.to_group then + return string.format("${dim}bridge<%s, %s>${clear}", v.from_group, v.to_group) + end + return string.format("${color.dump.string_quote}%s${clear}", v.name) + end}) + end + self._dag:dump() + + print("") + print("groups:") + for name, jobs in pairs(self._groups) do + print(" group(%s):", name) + for _, job in ipairs(jobs) do + cprint(" %s", job) + end + end + print("") +end + -- tostring function jobgraph:__tostring() return string.format("", self:name() or "anonymous", self:size()) -- cgit v1.3.1 From 1839ec9aebb573adc36803bf3d632f97410f3666 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 00:50:38 +0800 Subject: sort target rules --- tests/apis/rules_inject_deps/xmake.lua | 6 +----- tests/apis/rules_order/xmake.lua | 20 ++++++++++++++++-- xmake/actions/build/target_utils.lua | 37 ++++++++++++++++++++++++++-------- xmake/core/project/rule.lua | 5 +++++ xmake/modules/async/jobgraph.lua | 2 +- 5 files changed, 54 insertions(+), 16 deletions(-) (limited to 'xmake/modules') diff --git a/tests/apis/rules_inject_deps/xmake.lua b/tests/apis/rules_inject_deps/xmake.lua index cf73449bb..79b3f8252 100644 --- a/tests/apis/rules_inject_deps/xmake.lua +++ b/tests/apis/rules_inject_deps/xmake.lua @@ -1,10 +1,6 @@ rule("cppfront") set_extensions(".cpp2") - on_load(function (target) - local rule = target:rule("c++.build"):clone() - rule:add("deps", "cppfront", {order = true}) - target:rule_add(rule) - end) + add_buildorders("cppfront", "c++.build") on_build_file(function (target, sourcefile, opt) print("build cppfront file") local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp")) diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua index ea167dca9..8b3eb867a 100644 --- a/tests/apis/rules_order/xmake.lua +++ b/tests/apis/rules_order/xmake.lua @@ -1,7 +1,15 @@ rule("markdown") - add_deps("man", {order = true}) set_extensions(".md", ".markdown") + add_buildorders("man", "markdown") + + before_build(function (target) + print("before_build: markdown") + end) + after_build(function (target) + print("after_build: markdown") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -14,6 +22,14 @@ rule("markdown") rule("man") set_extensions(".man") + + before_build(function (target) + print("before_build: man") + end) + after_build(function (target) + print("after_build: man") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -26,7 +42,7 @@ rule("man") target("test") set_kind("binary") - add_rules("markdown") + add_rules("markdown", "man") add_files("src/*.c") add_files("src/*.md") add_files("src/*.man") diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 36c087548..9ee7a9c0d 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -117,7 +117,9 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false local job_prefix = target:fullname() - if target ~= instance then + if target == instance then + job_prefix = job_prefix .. "/target" + else job_prefix = job_prefix .. "/rule/" .. instance:fullname() end @@ -189,13 +191,11 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - -- TODO sort rules and jobs + -- call target and rules script local instances = {target} for _, r in ipairs(target:orderules()) do table.insert(instances, r) end - - -- call target and rules script local jobsize = jobgraph:size() jobgraph:group(group_name, function () local has_script = false @@ -204,9 +204,12 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do - if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then - has_script = true - end + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then + has_script = true + end + end) -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules if has_script and instance == target and stage == "" then break @@ -219,6 +222,23 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) + -- sort build rules + for _, instance in ipairs(instances) do + local buildorders = table.wrap(instance:get("buildorders")) + for _, buildorder in ipairs(buildorders) do + local joborders = {} + for _, name in ipairs(buildorder) do + local script_group = group_name .. "/" .. name + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) + end + end + end + if jobgraph:size() > jobsize then return group_name end @@ -298,6 +318,7 @@ function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) local dep = project.target(depname, {namespace = target:namespace()}) add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + -- build.across_targets_in_parallel is deprecated if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then jobname = string.format("%s/begin_%s", target:fullname(), job_kind) jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) @@ -329,7 +350,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local job_prefix = target:fullname() local file_group = sourcebatch.rulename if target == instance then - job_prefix = job_prefix .. "/" .. file_group + job_prefix = job_prefix .. "/target/" .. file_group else job_prefix = job_prefix .. "/rule/" .. file_group end diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index 63f954580..a2fd661e8 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -280,6 +280,11 @@ function rule.apis() , "rule.add_deps" , "rule.add_imports" } + , groups = + { + -- rule.add_xxx + "rule.add_buildorders" + } , script = { -- rule.on_xxx diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index be1ad57f0..4bbc53ff4 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -109,7 +109,7 @@ end -- has the given job or group? function jobgraph:has(name) - return self._jobs[name] or self._groups[name] + return (self._jobs[name] or self._groups[name]) ~= nil end -- enter group to add jobs -- cgit v1.3.1 From e6a81c6a5c84842d6f5b54cfd6d4cdd97f45eef9 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 18:35:59 +0800 Subject: rename to add_orders --- tests/apis/rules_inject_deps/xmake.lua | 2 +- tests/apis/rules_order/xmake.lua | 2 +- xmake/actions/build/target_utils.lua | 35 +++++++++++++++------------------- xmake/core/project/rule.lua | 2 +- xmake/modules/private/utils/rule.lua | 34 +++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 23 deletions(-) create mode 100644 xmake/modules/private/utils/rule.lua (limited to 'xmake/modules') diff --git a/tests/apis/rules_inject_deps/xmake.lua b/tests/apis/rules_inject_deps/xmake.lua index 79b3f8252..880bcde32 100644 --- a/tests/apis/rules_inject_deps/xmake.lua +++ b/tests/apis/rules_inject_deps/xmake.lua @@ -1,6 +1,6 @@ rule("cppfront") set_extensions(".cpp2") - add_buildorders("cppfront", "c++.build") + add_orders("cppfront", "c++.build") on_build_file(function (target, sourcefile, opt) print("build cppfront file") local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp")) diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua index 8b3eb867a..e05dc7bd2 100644 --- a/tests/apis/rules_order/xmake.lua +++ b/tests/apis/rules_order/xmake.lua @@ -1,7 +1,7 @@ rule("markdown") set_extensions(".md", ".markdown") - add_buildorders("man", "markdown") + add_orders("man", "markdown") before_build(function (target) print("before_build: markdown") diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 9ee7a9c0d..4c33b5adf 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -27,14 +27,7 @@ import("core.project.project") import("async.runjobs", {alias = "async_runjobs"}) import("async.jobgraph", {alias = "async_jobgraph"}) import("private.utils.batchcmds") - --- get rule --- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target -function _get_rule(target, rulename) - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - return ruleinst -end +import("private.utils.rule", {alias = "rule_utils"}) -- clean target for rebuilding function _clean_target(target) @@ -222,19 +215,21 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) - -- sort build rules + -- sort build rules, TODO deps for _, instance in ipairs(instances) do - local buildorders = table.wrap(instance:get("buildorders")) - for _, buildorder in ipairs(buildorders) do - local joborders = {} - for _, name in ipairs(buildorder) do - local script_group = group_name .. "/" .. name - if jobgraph:has(script_group) then - table.insert(joborders, script_group) + local orders = table.wrap(instance:get("orders")) + if #orders > 0 then + for _, order in ipairs(orders) do + local joborders = {} + for _, rulename in ipairs(order) do + local script_group = group_name .. "/" .. rulename + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) end - end - if #joborders > 0 then - jobgraph:add_orders(joborders) end end end @@ -503,7 +498,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, sourcebatch in pairs(sourcebatches) do local rulename = sourcebatch.rulename if rulename then - local ruleinst = _get_rule(target, rulename) + local ruleinst = rule_utils.get_rule(target, rulename) sourcebatches_map[ruleinst] = sourcebatch -- avoid duplicate scripts being called twice in the target, -- we just build sourcebatch with on_build_files scripts diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index a2fd661e8..ccab257c7 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -283,7 +283,7 @@ function rule.apis() , groups = { -- rule.add_xxx - "rule.add_buildorders" + "rule.add_orders" } , script = { diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua new file mode 100644 index 000000000..19706ad04 --- /dev/null +++ b/xmake/modules/private/utils/rule.lua @@ -0,0 +1,34 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file rule.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") + +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + -- cgit v1.3.1 From 529f59f911ddba8a2ef015f50df7a9582dd2bdb4 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 22:07:22 +0800 Subject: sort file rules --- xmake/actions/build/target_utils.lua | 64 ++++++++++++++++-------------------- xmake/modules/private/utils/rule.lua | 36 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 36 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 4c33b5adf..3d71ff55e 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -215,28 +215,14 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end end) - -- sort build rules, TODO deps - for _, instance in ipairs(instances) do - local orders = table.wrap(instance:get("orders")) - if #orders > 0 then - for _, order in ipairs(orders) do - local joborders = {} - for _, rulename in ipairs(order) do - local script_group = group_name .. "/" .. rulename - if jobgraph:has(script_group) then - table.insert(joborders, script_group) - end - end - if #joborders > 0 then - jobgraph:add_orders(joborders) - end - end - end + -- no any new jobs + if jobgraph:size() == jobsize then + return end - if jobgraph:size() > jobsize then - return group_name - end + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name end -- add target jobs for the given target @@ -517,8 +503,6 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) end end - -- TODO sort rules and jobs - -- call target and rules script local jobsize = jobgraph:size() jobgraph:group(group_name, function () @@ -530,26 +514,34 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) } local has_target_script = false for _, instance in ipairs(instances) do - if instance == target then - for _, sourcebatch in ipairs(sourcebatches_for_target) do - local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules - if has_script and stage == "" then - has_target_script = true + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if instance == target then + for _, sourcebatch in ipairs(sourcebatches_for_target) do + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end + end + elseif not has_target_script then -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch then + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) end end - elseif not has_target_script then -- rule - local sourcebatch = sourcebatches_map[instance] - if sourcebatch then - add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - end - end + end) end end) - if jobgraph:size() > jobsize then - return group_name + -- no any new jobs + if jobgraph:size() == jobsize then + return end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name end -- add file jobs for the given target diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua index 19706ad04..531486f61 100644 --- a/xmake/modules/private/utils/rule.lua +++ b/xmake/modules/private/utils/rule.lua @@ -32,3 +32,39 @@ function get_rule(target, rulename) return ruleinst end +-- build rules orders in jobgraph, we need to add rule job with groups +-- +-- like this: +-- @code +-- local root_group = "" +-- for _, ruleinst in ipairs(rules) do +-- local script_group = root_group .. "/" .. ruleinst:fullname() +-- jobgraph:group(script_group, function () +-- jobgraph:add("xxx", function (index, total, opt) +-- -- call rule script +-- end) +-- end) +-- end +-- +function build_orders_in_jobgraph(jobgraph, rules, opt) + opt = opt or {} + local root_group = assert(opt.root_group) + for _, ruleinst in ipairs(rules) do + local orders = table.wrap(ruleinst:get("orders")) + if #orders > 0 then + for _, order in ipairs(orders) do + local joborders = {} + for _, rulename in ipairs(order) do + local script_group = root_group .. "/" .. rulename + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) + end + end + end + end +end + -- cgit v1.3.1 From 54227eb4e653a46b8eb9ad9642d7641db052288b Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 21:38:39 +0800 Subject: fix progress --- xmake/actions/build/build.lua | 2 ++ xmake/actions/build/build_files.lua | 2 ++ xmake/actions/build/target_utils.lua | 4 ++-- xmake/modules/async/runjobs.lua | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 22097669a..d3fdd8e80 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -30,6 +30,7 @@ import("deprecated.build", {alias = "deprecated_build"}) function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" + opt.progress_factor = 0.05 target_utils.run_targetjobs(targets_root, opt) end @@ -37,6 +38,7 @@ end function _build(targets_root, opt) opt = opt or {} opt.job_kind = "build" + opt.progress_factor = 0.95 if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index e26c11aae..854f7de3f 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -75,6 +75,7 @@ end function _prepare_files(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" + opt.progress_factor = 0.05 opt.filepatterns = _get_file_patterns(opt.sourcefiles) target_utils.run_filejobs(targets_root, opt) end @@ -83,6 +84,7 @@ end function _build_files(targets_root, opt) opt = opt or {} opt.job_kind = "build" + opt.progress_factor = 0.95 opt.filepatterns = _get_file_patterns(opt.sourcefiles) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 92723d46c..147c25187 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -686,7 +686,7 @@ function run_targetjobs(targets_root, opt) if errors and progress.showing_without_scroll() then print("") end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) os.cd(curdir) return true end @@ -704,7 +704,7 @@ function run_filejobs(targets_root, opt) if errors and progress.showing_without_scroll() then print("") end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc}) + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) os.cd(curdir) return true end diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index d2af72843..adf68018d 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -163,6 +163,7 @@ function main(name, jobs, opt) local abort_errors local progress_wrapper = {} local job_pending + local progress_factor = opt.progress_factor or 1.0 progress_wrapper.current = function () return count end @@ -171,7 +172,7 @@ function main(name, jobs, opt) end progress_wrapper.percent = function () if total and total > 0 then - return math.floor((count * 100) / total) + return math.floor((count * progress_factor * 100) / total) else return 0 end -- cgit v1.3.1 From 6d8671865bbe9026305186c54b4ef82ebb80a4bc Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 22:11:00 +0800 Subject: enable jobgraph for pch --- xmake/modules/private/action/build/object.lua | 1 + xmake/modules/private/action/build/pcheader.lua | 6 +++--- xmake/rules/c++/precompiled_header/xmake.lua | 12 ++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 0028b4b6f..a8b66aa54 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -174,6 +174,7 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) end function main(target, jobgraph, sourcebatch, opt) + opt = opt or {} if jobgraph.add_orders then _add_jobgraph(target, jobgraph, sourcebatch, opt) else diff --git a/xmake/modules/private/action/build/pcheader.lua b/xmake/modules/private/action/build/pcheader.lua index 20e857d7f..ec06c8fb8 100644 --- a/xmake/modules/private/action/build/pcheader.lua +++ b/xmake/modules/private/action/build/pcheader.lua @@ -20,7 +20,7 @@ -- imports import("core.language.language") -import("object") +import("object", {alias = "build_objects"}) function config(target, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) @@ -55,7 +55,7 @@ function config(target, langkind, opt) end -- add batch jobs to build the precompiled header file -function build(target, langkind, opt) +function build(target, jobgraph, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) if pcheaderfile then local sourcefile = pcheaderfile @@ -63,6 +63,6 @@ function build(target, langkind, opt) local dependfile = target:dependfile(objectfile) local sourcekind = language.langkinds()[langkind] local sourcebatch = {sourcekind = sourcekind, sourcefiles = {sourcefile}, objectfiles = {objectfile}, dependfiles = {dependfile}} - object.build(target, sourcebatch, opt) + build_objects(target, jobgraph, sourcebatch, opt) end end diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 3f30c1ea1..08ecf3fd7 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -23,16 +23,16 @@ rule("c.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) - on_prepare(function (target, opt) - import("private.action.build.pcheader").build(target, "c", opt) - end) + on_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "c", opt) + end, {jobgraph = true}) rule("c++.build.pcheader") add_orders("c++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) - on_prepare(function (target, opt) - import("private.action.build.pcheader").build(target, "cxx", opt) - end) + on_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "cxx", opt) + end, {jobgraph = true}) -- cgit v1.3.1 From 07ac1c481e0a30161389328365efcbebc15d88d9 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 7 Apr 2025 22:40:03 +0800 Subject: fix distcc --- xmake/modules/async/jobgraph.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index 4bbc53ff4..1ed5873b7 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -73,7 +73,7 @@ function jobgraph:add(name, run, opt) local dag = self._dag local jobs = self._jobs if not jobs[name] then - local job = {name = name, run = run, opt = opt} + local job = {name = name, run = run, distcc = opt.distcc} jobs[name] = job dag:add_vertex(job) self._size = self._size + 1 -- cgit v1.3.1 From 6fc28a78bf95490b5c230a42c0f4a40ab5aad2e4 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:40:15 +0800 Subject: move target files --- xmake/actions/build/build.lua | 2 +- xmake/actions/build/build_files.lua | 2 +- xmake/actions/build/target_utils.lua | 717 -------------------- .../modules/private/action/build/build_binary.lua | 38 ++ .../private/action/build/build_moduleonly.lua | 26 + .../modules/private/action/build/build_object.lua | 26 + .../modules/private/action/build/build_shared.lua | 26 + .../modules/private/action/build/build_static.lua | 26 + .../modules/private/action/build/link_objects.lua | 68 ++ .../modules/private/action/build/prepare_files.lua | 27 + xmake/modules/private/action/build/target.lua | 718 +++++++++++++++++++++ 11 files changed, 957 insertions(+), 719 deletions(-) delete mode 100644 xmake/actions/build/target_utils.lua create mode 100644 xmake/modules/private/action/build/build_binary.lua create mode 100644 xmake/modules/private/action/build/build_moduleonly.lua create mode 100644 xmake/modules/private/action/build/build_object.lua create mode 100644 xmake/modules/private/action/build/build_shared.lua create mode 100644 xmake/modules/private/action/build/build_static.lua create mode 100644 xmake/modules/private/action/build/link_objects.lua create mode 100644 xmake/modules/private/action/build/prepare_files.lua create mode 100644 xmake/modules/private/action/build/target.lua (limited to 'xmake/modules') diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index d3fdd8e80..270b24ea7 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -22,8 +22,8 @@ import("core.base.option") import("core.project.config") import("core.project.project") -import("target_utils") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_utils"}) import("deprecated.build", {alias = "deprecated_build"}) -- run prepare jobs diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 854f7de3f..12d9b00ee 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -23,8 +23,8 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") -import("target_utils") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_utils"}) import("deprecated.build_files", {alias = "deprecated_build_files"}) -- convert all sourcefiles to lua pattern diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua deleted file mode 100644 index 2f68830c0..000000000 --- a/xmake/actions/build/target_utils.lua +++ /dev/null @@ -1,717 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file target_utils.lua --- - --- imports -import("core.base.option") -import("core.base.hashset") -import("core.project.rule") -import("core.project.config") -import("core.project.project") -import("async.runjobs", {alias = "async_runjobs"}) -import("async.jobgraph", {alias = "async_jobgraph"}) -import("private.utils.batchcmds") -import("private.utils.rule", {alias = "rule_utils"}) - --- clean target for rebuilding -function _clean_target(target) - if target:targetfile() then - os.tryrm(target:symbolfile()) - os.tryrm(target:targetfile()) - end -end - --- match source files -function _match_sourcefiles(sourcefile, filepatterns) - for _, filepattern in ipairs(filepatterns) do - if sourcefile:match(filepattern.pattern) == sourcefile then - if filepattern.excludes then - if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then - sourcefile = sourcefile:sub(#filepattern.rootdir + 2) - end - for _, exclude in ipairs(filepattern.excludes) do - if sourcefile:match(exclude) == sourcefile then - return false - end - end - end - return true - end - end -end - --- match sourcebatches -function _match_sourcebatches(target, filepatterns) - local newbatches = {} - local sourcecount = 0 - for rulename, sourcebatch in pairs(target:sourcebatches()) do - local objectfiles = sourcebatch.objectfiles - local dependfiles = sourcebatch.dependfiles - local sourcekind = sourcebatch.sourcekind - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do - if _match_sourcefiles(sourcefile, filepatterns) then - local newbatch = newbatches[rulename] - if not newbatch then - newbatch = {} - newbatch.sourcekind = sourcekind - newbatch.rulename = rulename - newbatch.sourcefiles = {} - end - table.insert(newbatch.sourcefiles, sourcefile) - if objectfiles then - newbatch.objectfiles = newbatch.objectfiles or {} - table.insert(newbatch.objectfiles, objectfiles[idx]) - end - if dependfiles then - newbatch.dependfiles = newbatch.dependfiles or {} - table.insert(newbatch.dependfiles, dependfiles[idx]) - end - newbatches[rulename] = newbatch - sourcecount = sourcecount + 1 - end - end - end - if sourcecount > 0 then - return newbatches - end -end - --- add targetjobs and deps orders -function _add_targetjobs_orders(jobgraph, target, dep, opt) - local jobname, jobname_dep - local job_kind = opt.job_kind - if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - -- build.across_targets_in_parallel is deprecated - if dep:policy("build.across_targets_in_parallel") == false then - wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") - end - elseif job_kind == "build" then - jobname = target:fullname() .. "/link" - jobname_dep = dep:fullname() .. "/link" - if not jobgraph:has(jobname) then - jobname = string.format("%s/begin_%s", target:fullname(), job_kind) - end - if not jobgraph:has(jobname_dep) then - jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) - end - end - if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then - jobgraph:add_orders(jobname_dep, jobname) - end -end - --- add target jobs for the builtin script -function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) - if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then - if job_kind == "prepare" then - import("builtin.prepare_files", {anonymous = true})(jobgraph, target) - elseif job_kind == "link" then - import("builtin.link_objects", {anonymous = true})(jobgraph, target) - else - import("builtin.build_" .. target:kind(), {anonymous = true})(jobgraph, target) - end - end -end - --- add target jobs for the given script -function add_targetjobs_for_script(jobgraph, target, instance, opt) - opt = opt or {} - local has_script = false - local job_prefix = target:fullname() - if target == instance then - job_prefix = job_prefix .. "/target" - else - job_prefix = job_prefix .. "/rule/" .. instance:fullname() - end - - -- call script - if not has_script then - local script_name = opt.script_name - local script = instance:script(script_name) - if script then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build(function (target, jobgraph, opt) - -- end, {jobgraph = true}) - if instance:extraconf(script_name, "jobgraph") then - script(target, jobgraph) - elseif instance:extraconf(script_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_name) - jobgraph:add(jobname, function (index, total, opt) - script(target, {progress = opt.progress}) - end) - end - has_script = true - end - end - - -- call command script - -- e.g. - -- - -- target("test") - -- on_buildcmd(function (target, batchcmds, opt) - -- end) - if not has_script then - local scriptcmd_name = opt.scriptcmd_name - local scriptcmd = instance:script(scriptcmd_name) - if scriptcmd then - local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - return has_script -end - --- add target jobs with the given stage --- stage: before, after or "" -function add_targetjobs_with_stage(jobgraph, target, stage, opt) - opt = opt or {} - local job_kind = opt.job_kind - - -- the group name, e.g. foo/after_prepare, bar/before_build - local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) - - -- the script name, e.g. before/after_prepare, before/after_build - local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind - - -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd - local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") - - -- call target and rules script - local instances = {target} - for _, r in ipairs(target:orderules()) do - table.insert(instances, r) - end - local jobsize = jobgraph:size() - jobgraph:group(group_name, function () - local has_script = false - local script_opt = { - script_name = script_name, - scriptcmd_name = scriptcmd_name - } - for _, instance in ipairs(instances) do - local script_group = group_name .. "/" .. instance:fullname() - jobgraph:group(script_group, function () - if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then - has_script = true - end - end) - -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules - if has_script and instance == target and stage == "" then - break - end - end - - -- call builtin script, e.g. on_prepare, on_build, ... - if not has_script and stage == "" then - add_targetjobs_for_builtin_script(jobgraph, target, job_kind) - end - end) - - -- no any new jobs - if jobgraph:size() == jobsize then - return - end - - -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) - return group_name -end - --- add target jobs for the given target -function add_targetjobs(jobgraph, target, opt) - opt = opt or {} - if not target:is_enabled() then - return - end - - local pkgenvs = _g.pkgenvs - if pkgenvs == nil then - pkgenvs = {} - _g.pkgenvs = pkgenvs - end - - local job_kind = opt.job_kind - local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) - local job_end = string.format("%s/end_%s", target:fullname(), job_kind) - jobgraph:add(job_begin, function (index, total, opt) - -- enter package environments - -- https://github.com/xmake-io/xmake/issues/4033 - -- - -- maybe mixing envs isn't a great solution, - -- but it's the most efficient compromise compared to setting envs in every on_build_file. - -- - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = target:pkgenvs() - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - -- clean target first if rebuild - if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then - _clean_target(target) - end - end) - - jobgraph:add(job_end, function (index, total, opt) - -- restore environments - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = nil - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - end) - - -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) - jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) -end - --- add target jobs for the given target and deps -function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) - local targetname = target:fullname() - if not targetrefs[targetname] then - targetrefs[targetname] = target - add_targetjobs(jobgraph, target, opt) - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) - _add_targetjobs_orders(jobgraph, target, dep, opt) - end - end -end - --- get target jobs -function get_targetjobs(targets_root, opt) - local jobgraph = async_jobgraph.new(opt.job_kind) - local targetrefs = {} - for _, target in ipairs(targets_root) do - add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) - end - return jobgraph -end - --- add file jobs for the given script -function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) - opt = opt or {} - local has_script = false - local job_prefix = target:fullname() - local file_group = sourcebatch.rulename - if target == instance then - job_prefix = job_prefix .. "/target/" .. file_group - else - job_prefix = job_prefix .. "/rule/" .. file_group - end - - -- call script files - if not has_script then - local script_files_name = opt.script_files_name - local script_files = instance:script(script_files_name) - if script_files then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build_files(function (target, jobgraph, sourcebatch, opt) - -- end, {jobgraph = true}) - local distcc = instance:extraconf(script_files_name, "distcc") - if instance:extraconf(script_files_name, "jobgraph") then - script_files(target, jobgraph, sourcebatch, {distcc = distcc}) - elseif instance:extraconf(script_files_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", - instance:fullname(), script_files_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build_files(function (target, sourcebatch, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_files_name) - jobgraph:add(jobname, function (index, total, opt) - script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) - end) - end - has_script = true - end - end - - -- call script file - if not has_script then - local script_file_name = opt.script_file_name - local script_file = instance:script(script_file_name) - if script_file then - -- call custom script with jobgraph - -- e.g. - -- - -- target("test") - -- on_build_file(function (target, jobgraph, sourcefile, opt) - -- end, {jobgraph = true}) - local distcc = instance:extraconf(script_file_name, "distcc") - if instance:extraconf(script_file_name, "jobgraph") then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) - end - elseif instance:extraconf(script_file_name, "batch") then - wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", - instance:fullname(), script_file_name) - else - -- call custom script directly - -- e.g. - -- - -- target("test") - -- on_build_file(function (target, sourcefile, opt) - -- end) - local jobname = string.format("%s/%s", job_prefix, script_file_name) - jobgraph:add(jobname, function (index, total, opt) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) - end - end) - end - has_script = true - end - end - - -- call command script files - -- e.g. - -- - -- target("test") - -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) - -- end) - if not has_script then - local scriptcmd_files_name = opt.scriptcmd_files_name - local scriptcmd_files = instance:script(scriptcmd_files_name) - if scriptcmd_files then - local distcc = instance:extraconf(scriptcmd_files_name, "distcc") - local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - - -- call command script file - -- e.g. - -- - -- target("test") - -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) - -- end) - if not has_script then - local scriptcmd_file_name = opt.scriptcmd_file_name - local scriptcmd_file = instance:script(scriptcmd_file_name) - if scriptcmd_file then - local distcc = instance:extraconf(scriptcmd_file_name, "distcc") - local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) - jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) - end - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - has_script = true - end - end - return has_script -end - --- add file jobs with the given stage --- stage: before, after or "" --- -function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) - opt = opt or {} - local job_kind = opt.job_kind - local job_kind_file = job_kind .. "_file" - local job_kind_files = job_kind .. "_files" - local job_kindcmd_file = job_kind .. "cmd_file" - local job_kindcmd_files = job_kind .. "cmd_files" - - -- the group name, e.g. foo/after_prepare_files, bar/before_build_files - local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) - - -- the script name, e.g. before/after_prepare_files, before/after_build_files - local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file - local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files - - -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files - local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file - local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files - - -- build sourcebatches map - local instances = {target} - local sourcebatches_map = {} - local sourcebatches_for_target = {} - for _, sourcebatch in pairs(sourcebatches) do - local rulename = sourcebatch.rulename - if rulename then - local ruleinst = rule_utils.get_rule(target, rulename) - sourcebatches_map[ruleinst] = sourcebatch - -- avoid duplicate scripts being called twice in the target, - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - if ruleinst:script("build_file") or ruleinst:script("build_files") then - table.insert(sourcebatches_for_target, sourcebatch) - end - table.insert(instances, ruleinst) - else - table.insert(sourcebatches_for_target, sourcebatch) - end - end - - -- call target and rules script - local jobsize = jobgraph:size() - jobgraph:group(group_name, function () - local script_opt = { - script_file_name = script_file_name, - script_files_name = script_files_name, - scriptcmd_file_name = scriptcmd_file_name, - scriptcmd_files_name = scriptcmd_files_name - } - local has_target_script = false - for _, instance in ipairs(instances) do - local script_group = group_name .. "/" .. instance:fullname() - jobgraph:group(script_group, function () - if instance == target then - for _, sourcebatch in ipairs(sourcebatches_for_target) do - local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules - if has_script and stage == "" then - has_target_script = true - end - end - elseif not has_target_script then -- rule - local sourcebatch = sourcebatches_map[instance] - if sourcebatch then - add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) - end - end - end) - end - end) - - -- no any new jobs - if jobgraph:size() == jobsize then - return - end - - -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) - return group_name -end - --- add file jobs for the given target -function add_filejobs(jobgraph, target, opt) - opt = opt or {} - if not target:is_enabled() then - return - end - - -- get sourcebatches - local sourcebatches - local filepatterns = opt.filepatterns - if filepatterns then - sourcebatches = _match_sourcebatches(target, filepatterns) - else - sourcebatches = target:sourcebatches() - end - - -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) - local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) - local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) - jobgraph:add_orders(group_before, group, group_after) -end - --- add file jobs for the given target and deps -function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) - local targetname = target:fullname() - if not targetrefs[targetname] then - targetrefs[targetname] = target - add_filejobs(jobgraph, target, opt) - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) - end - end -end - --- get files jobs -function get_filejobs(targets_root, opt) - local jobgraph = async_jobgraph.new(opt.job_kind) - local targetrefs = {} - for _, target in ipairs(targets_root) do - add_filejobs_and_deps(jobgraph, target, targetrefs, opt) - end - return jobgraph -end - --- add link jobs for the given target -function add_linkjobs(jobgraph, target, opt) - opt = opt or {} - opt.job_kind = "link" - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) - jobgraph:add_orders(group_before, group, group_after) -end - --- get link depfiles -function get_linkdepfiles(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles -end - --- get all root targets -function get_root_targets(targetnames, opt) - opt = opt or {} - - -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local group_pattern = opt.group_pattern - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end - return targets_root -end - --- run target-level jobs, e.g. on_prepare, on_build, ... -function run_targetjobs(targets_root, opt) - opt = opt or {} - local job_kind = opt.job_kind - local jobgraph = get_targetjobs(targets_root, opt) - if jobgraph and not jobgraph:empty() then - local curdir = os.curdir() - async_runjobs(job_kind, jobgraph, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) - os.cd(curdir) - return true - end -end - --- run files-level jobs, e.g. on_prepare_files, on_build_files, ... -function run_filejobs(targets_root, opt) - opt = opt or {} - local job_kind = opt.job_kind - local jobgraph = get_filejobs(targets_root, opt) - if jobgraph and not jobgraph:empty() then - local curdir = os.curdir() - async_runjobs(job_kind, jobgraph, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) - os.cd(curdir) - return true - end -end diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua new file mode 100644 index 000000000..c30addfef --- /dev/null +++ b/xmake/modules/private/action/build/build_binary.lua @@ -0,0 +1,38 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_binary.lua +-- + +-- imports +import("build_object") +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + local objects_group = target:fullname() .. "/objects" + local jobsize = jobgraph:size() + jobgraph:group(objects_group, function () + build_object(jobgraph, target) + end) + if jobgraph:size() > jobsize then + local link_group = target:fullname() .. "/link" + jobgraph:group(link_group, function () + target_utils.add_linkjobs(jobgraph, target) + end) + jobgraph:add_orders(objects_group, link_group) + end +end diff --git a/xmake/modules/private/action/build/build_moduleonly.lua b/xmake/modules/private/action/build/build_moduleonly.lua new file mode 100644 index 000000000..e3cc87c27 --- /dev/null +++ b/xmake/modules/private/action/build/build_moduleonly.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_moduleonly.lua +-- + +-- imports +import("build_object") + +function main(jobgraph, target) + build_object(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua new file mode 100644 index 000000000..2a44bcea6 --- /dev/null +++ b/xmake/modules/private/action/build/build_object.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_object.lua +-- + +-- imports +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) +end diff --git a/xmake/modules/private/action/build/build_shared.lua b/xmake/modules/private/action/build/build_shared.lua new file mode 100644 index 000000000..077489710 --- /dev/null +++ b/xmake/modules/private/action/build/build_shared.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_shared.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target) + build_binary(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/build_static.lua b/xmake/modules/private/action/build/build_static.lua new file mode 100644 index 000000000..8b9b2ffaf --- /dev/null +++ b/xmake/modules/private/action/build/build_static.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_static.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target) + build_binary(jobgraph, target) +end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua new file mode 100644 index 000000000..099674d3e --- /dev/null +++ b/xmake/modules/private/action/build/link_objects.lua @@ -0,0 +1,68 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file link_objects.lua +-- + +-- imports +import("core.base.option") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("build_object") +import("private.action.build.target", {alias = "target_utils"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = target_utils.get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +function main(jobgraph, target) + local linkjob = target:fullname() .. "/link_objects" + jobgraph:add(linkjob, function (index, total, opt) + _do_link_target(target, opt) + end) +end diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua new file mode 100644 index 000000000..a0ecc242a --- /dev/null +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -0,0 +1,27 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_files.lua +-- + +-- imports +import("core.base.option") +import("private.action.build.target", {alias = "target_utils"}) + +function main(jobgraph, target) + target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) +end diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua new file mode 100644 index 000000000..b6d428692 --- /dev/null +++ b/xmake/modules/private/action/build/target.lua @@ -0,0 +1,718 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file target.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.rule") +import("core.project.config") +import("core.project.project") +import("async.runjobs", {alias = "async_runjobs"}) +import("async.jobgraph", {alias = "async_jobgraph"}) +import("private.utils.batchcmds") +import("private.utils.rule", {alias = "rule_utils"}) + +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- match sourcebatches +function _match_sourcebatches(target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return newbatches + end +end + +-- add targetjobs and deps orders +function _add_targetjobs_orders(jobgraph, target, dep, opt) + local jobname, jobname_dep + local job_kind = opt.job_kind + if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + -- build.across_targets_in_parallel is deprecated + if dep:policy("build.across_targets_in_parallel") == false then + wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") + end + elseif job_kind == "build" then + jobname = target:fullname() .. "/link" + jobname_dep = dep:fullname() .. "/link" + if not jobgraph:has(jobname) then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + end + if not jobgraph:has(jobname_dep) then + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + end + end + if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then + jobgraph:add_orders(jobname_dep, jobname) + end +end + +-- add target jobs for the builtin script +function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then + if job_kind == "prepare" then + import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target) + elseif job_kind == "link" then + import("private.action.build.link_objects", {anonymous = true})(jobgraph, target) + else + import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target) + end + end +end + +-- add target jobs for the given script +function add_targetjobs_for_script(jobgraph, target, instance, opt) + opt = opt or {} + local has_script = false + local job_prefix = target:fullname() + if target == instance then + job_prefix = job_prefix .. "/target" + else + job_prefix = job_prefix .. "/rule/" .. instance:fullname() + end + + -- call script + if not has_script then + local script_name = opt.script_name + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true + end + end + + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + if not has_script then + local scriptcmd_name = opt.scriptcmd_name + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add target jobs with the given stage +-- stage: before, after or "" +function add_targetjobs_with_stage(jobgraph, target, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + + -- the group name, e.g. foo/after_prepare, bar/before_build + local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare, before/after_build + local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind + + -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd + local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") + + -- call target and rules script + local instances = {target} + for _, r in ipairs(target:orderules()) do + table.insert(instances, r) + end + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local has_script = false + local script_opt = { + script_name = script_name, + scriptcmd_name = scriptcmd_name + } + for _, instance in ipairs(instances) do + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then + has_script = true + end + end) + -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules + if has_script and instance == target and stage == "" then + break + end + end + + -- call builtin script, e.g. on_prepare, on_build, ... + if not has_script and stage == "" then + add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name +end + +-- add target jobs for the given target +function add_targetjobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + local pkgenvs = _g.pkgenvs + if pkgenvs == nil then + pkgenvs = {} + _g.pkgenvs = pkgenvs + end + + local job_kind = opt.job_kind + local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) + local job_end = string.format("%s/end_%s", target:fullname(), job_kind) + jobgraph:add(job_begin, function (index, total, opt) + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target first if rebuild + if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + end) + + jobgraph:add(job_end, function (index, total, opt) + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + end) + + -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) +end + +-- add target jobs for the given target and deps +function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_targetjobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + _add_targetjobs_orders(jobgraph, target, dep, opt) + end + end +end + +-- get target jobs +function get_targetjobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add file jobs for the given script +function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) + opt = opt or {} + local has_script = false + local job_prefix = target:fullname() + local file_group = sourcebatch.rulename + if target == instance then + job_prefix = job_prefix .. "/target/" .. file_group + else + job_prefix = job_prefix .. "/rule/" .. file_group + end + + -- call script files + if not has_script then + local script_files_name = opt.script_files_name + local script_files = instance:script(script_files_name) + if script_files then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, jobgraph, sourcebatch, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_files_name, "distcc") + if instance:extraconf(script_files_name, "jobgraph") then + script_files(target, jobgraph, sourcebatch, {distcc = distcc}) + elseif instance:extraconf(script_files_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_files_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, sourcebatch, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_files_name) + jobgraph:add(jobname, function (index, total, opt) + script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) + end) + end + has_script = true + end + end + + -- call script file + if not has_script then + local script_file_name = opt.script_file_name + local script_file = instance:script(script_file_name) + if script_file then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, jobgraph, sourcefile, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_file_name, "distcc") + if instance:extraconf(script_file_name, "jobgraph") then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) + end + elseif instance:extraconf(script_file_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_file_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, sourcefile, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_file_name) + jobgraph:add(jobname, function (index, total, opt) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + end) + end + has_script = true + end + end + + -- call command script files + -- e.g. + -- + -- target("test") + -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- end) + if not has_script then + local scriptcmd_files_name = opt.scriptcmd_files_name + local scriptcmd_files = instance:script(scriptcmd_files_name) + if scriptcmd_files then + local distcc = instance:extraconf(scriptcmd_files_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + + -- call command script file + -- e.g. + -- + -- target("test") + -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) + -- end) + if not has_script then + local scriptcmd_file_name = opt.scriptcmd_file_name + local scriptcmd_file = instance:script(scriptcmd_file_name) + if scriptcmd_file then + local distcc = instance:extraconf(scriptcmd_file_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) + jobgraph:add(jobname, function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end) + has_script = true + end + end + return has_script +end + +-- add file jobs with the given stage +-- stage: before, after or "" +-- +function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + local job_kind_file = job_kind .. "_file" + local job_kind_files = job_kind .. "_files" + local job_kindcmd_file = job_kind .. "cmd_file" + local job_kindcmd_files = job_kind .. "cmd_files" + + -- the group name, e.g. foo/after_prepare_files, bar/before_build_files + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare_files, before/after_build_files + local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file + local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files + + -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files + local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file + local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files + + -- build sourcebatches map + local instances = {target} + local sourcebatches_map = {} + local sourcebatches_for_target = {} + for _, sourcebatch in pairs(sourcebatches) do + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = rule_utils.get_rule(target, rulename) + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + table.insert(instances, ruleinst) + else + table.insert(sourcebatches_for_target, sourcebatch) + end + end + + -- call target and rules script + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local script_opt = { + script_file_name = script_file_name, + script_files_name = script_files_name, + scriptcmd_file_name = scriptcmd_file_name, + scriptcmd_files_name = scriptcmd_files_name + } + local has_target_script = false + for _, instance in ipairs(instances) do + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if instance == target then + for _, sourcebatch in ipairs(sourcebatches_for_target) do + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end + end + elseif not has_target_script then -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch then + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + end + end + end) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + return group_name +end + +-- add file jobs for the given target +function add_filejobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + -- get sourcebatches + local sourcebatches + local filepatterns = opt.filepatterns + if filepatterns then + sourcebatches = _match_sourcebatches(target, filepatterns) + else + sourcebatches = target:sourcebatches() + end + + -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files + local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + jobgraph:add_orders(group_before, group, group_after) +end + +-- add file jobs for the given target and deps +function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_filejobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) + end + end +end + +-- get files jobs +function get_filejobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add link jobs for the given target +function add_linkjobs(jobgraph, target, opt) + opt = opt or {} + opt.job_kind = "link" + local group = add_targetjobs_with_stage(jobgraph, target, "", opt) + local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + jobgraph:add_orders(group_before, group, group_after) +end + +-- get link depfiles +function get_linkdepfiles(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end + +-- get all root targets +function get_root_targets(targetnames, opt) + opt = opt or {} + + -- get root targets + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + return targets_root +end + +-- run target-level jobs, e.g. on_prepare, on_build, ... +function run_targetjobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_targetjobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + +-- run files-level jobs, e.g. on_prepare_files, on_build_files, ... +function run_filejobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_filejobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + -- cgit v1.3.1 From f72d519dfe7b3a3744660ed15c620dbd5344c16c Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 8 Apr 2025 00:41:42 +0800 Subject: prepare targets in cmakelists generator --- xmake/actions/build/build.lua | 8 ++++---- xmake/actions/build/build_files.lua | 8 ++++---- xmake/modules/private/action/build/build_binary.lua | 4 ++-- xmake/modules/private/action/build/build_object.lua | 4 ++-- xmake/modules/private/action/build/link_objects.lua | 4 ++-- xmake/modules/private/action/build/prepare_files.lua | 4 ++-- xmake/plugins/project/cmake/cmakelists.lua | 19 ++++++++----------- 7 files changed, 24 insertions(+), 27 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 270b24ea7..d4f5658fb 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -23,7 +23,7 @@ import("core.base.option") import("core.project.config") import("core.project.project") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("deprecated.build", {alias = "deprecated_build"}) -- run prepare jobs @@ -31,7 +31,7 @@ function _prepare(targets_root, opt) opt = opt or {} opt.job_kind = "prepare" opt.progress_factor = 0.05 - target_utils.run_targetjobs(targets_root, opt) + target_buildutils.run_targetjobs(targets_root, opt) end -- run build jobs @@ -42,13 +42,13 @@ function _build(targets_root, opt) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end - target_utils.run_targetjobs(targets_root, opt) + target_buildutils.run_targetjobs(targets_root, opt) end function main(targetnames, opt) -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) + local targets_root = target_buildutils.get_root_targets(targetnames, opt) -- prepare to build _prepare(targets_root, opt) diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index 12d9b00ee..1bbad79f6 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -24,7 +24,7 @@ import("core.base.hashset") import("core.project.config") import("core.project.project") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("deprecated.build_files", {alias = "deprecated_build_files"}) -- convert all sourcefiles to lua pattern @@ -77,7 +77,7 @@ function _prepare_files(targets_root, opt) opt.job_kind = "prepare" opt.progress_factor = 0.05 opt.filepatterns = _get_file_patterns(opt.sourcefiles) - target_utils.run_filejobs(targets_root, opt) + target_buildutils.run_filejobs(targets_root, opt) end -- run build files jobs @@ -89,7 +89,7 @@ function _build_files(targets_root, opt) if distcc_build_client.is_connected() then opt.distcc = distcc_build_client.singleton() end - if not target_utils.run_filejobs(targets_root, opt) then + if not target_buildutils.run_filejobs(targets_root, opt) then wprint("%s not found!", opt.sourcefiles) end end @@ -97,7 +97,7 @@ end function main(targetnames, opt) -- get root targets - local targets_root = target_utils.get_root_targets(targetnames, opt) + local targets_root = target_buildutils.get_root_targets(targetnames, opt) -- prepare to build files _prepare_files(targets_root, opt) diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua index c30addfef..72263e6c8 100644 --- a/xmake/modules/private/action/build/build_binary.lua +++ b/xmake/modules/private/action/build/build_binary.lua @@ -20,7 +20,7 @@ -- imports import("build_object") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) local objects_group = target:fullname() .. "/objects" @@ -31,7 +31,7 @@ function main(jobgraph, target) if jobgraph:size() > jobsize then local link_group = target:fullname() .. "/link" jobgraph:group(link_group, function () - target_utils.add_linkjobs(jobgraph, target) + target_buildutils.add_linkjobs(jobgraph, target) end) jobgraph:add_orders(objects_group, link_group) end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua index 2a44bcea6..2a6baa59a 100644 --- a/xmake/modules/private/action/build/build_object.lua +++ b/xmake/modules/private/action/build/build_object.lua @@ -19,8 +19,8 @@ -- -- imports -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "build"}) + target_buildutils.add_filejobs(jobgraph, target, {job_kind = "build"}) end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua index 099674d3e..25214d27e 100644 --- a/xmake/modules/private/action/build/link_objects.lua +++ b/xmake/modules/private/action/build/link_objects.lua @@ -25,7 +25,7 @@ import("core.tool.compiler") import("core.project.depend") import("utils.progress") import("build_object") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -33,7 +33,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = target_utils.get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua index a0ecc242a..d8ca351ca 100644 --- a/xmake/modules/private/action/build/prepare_files.lua +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -20,8 +20,8 @@ -- imports import("core.base.option") -import("private.action.build.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) function main(jobgraph, target) - target_utils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) + target_buildutils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 22edabcd2..464101268 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -31,6 +31,7 @@ import("lib.detect.find_tool") import("private.utils.batchcmds") import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) +import("private.action.build.target", {alias = "target_buildutils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -298,6 +299,12 @@ function _get_flags_from_target(target, flagkind) return results end +-- prepare targets +function _prepare_targets() + local targets_root = target_buildutils.get_root_targets() + target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) +end + -- set compiler function _set_compiler(cmakelists) if config.get("toolchain") then @@ -1331,21 +1338,11 @@ function _generate_cmakelists(cmakelists, outputdir) end end --- make function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) - - -- open the cmakelists local cmakelists = io.open(path.join(outputdir, "CMakeLists.txt"), "w") - - -- generate cmakelists + _prepare_targets() _generate_cmakelists(cmakelists, outputdir) - - -- close the cmakelists cmakelists:close() - - -- leave project directory os.cd(oldir) end -- cgit v1.3.1 From 117ae687a27f34cf183e967e8688eb3d7e8e4f81 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 00:45:04 +0800 Subject: compatible with order deps --- xmake/core/project/rule.lua | 12 ++++++++++++ xmake/modules/private/action/build/target.lua | 6 ++++-- xmake/modules/private/utils/rule.lua | 6 ++++-- 3 files changed, 20 insertions(+), 4 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index ccab257c7..89ee45f8b 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -61,6 +61,18 @@ function _instance:_build_deps() self._DEPS = self._DEPS or {} self._ORDERDEPS = self._ORDERDEPS or {} instance_deps.load_deps(self, instances, self._DEPS, self._ORDERDEPS, {self:fullname()}) + + -- compatible with `add_deps("foo", {order = true})` + local plaindeps = self:get("deps") + if plaindeps then + for _, depname in ipairs(table.wrap(plaindeps)) do + if self:extraconf("deps", depname, "order") then + self:add("orders", depname, self:name()) + utils.warning("add_deps(%s, {order = true}) has been deprecated, please use `add_orders(%s, %s) instead of it`", + depname, depname, self:name()) + end + end + end end -- clone rule diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index b6d428692..beb6f9cb2 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -223,6 +223,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) scriptcmd_name = scriptcmd_name } for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders local script_group = group_name .. "/" .. instance:fullname() jobgraph:group(script_group, function () if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then @@ -247,7 +248,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) end -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) return group_name end @@ -526,6 +527,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) } local has_target_script = false for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders local script_group = group_name .. "/" .. instance:fullname() jobgraph:group(script_group, function () if instance == target then @@ -552,7 +554,7 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) end -- sort build rules - rule_utils.build_orders_in_jobgraph(jobgraph, instances, {root_group = group_name}) + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) return group_name end diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua index 531486f61..ed6aec451 100644 --- a/xmake/modules/private/utils/rule.lua +++ b/xmake/modules/private/utils/rule.lua @@ -46,7 +46,7 @@ end -- end) -- end -- -function build_orders_in_jobgraph(jobgraph, rules, opt) +function build_orders_in_jobgraph(jobgraph, target, rules, opt) opt = opt or {} local root_group = assert(opt.root_group) for _, ruleinst in ipairs(rules) do @@ -55,7 +55,9 @@ function build_orders_in_jobgraph(jobgraph, rules, opt) for _, order in ipairs(orders) do local joborders = {} for _, rulename in ipairs(order) do - local script_group = root_group .. "/" .. rulename + -- we need to use fullname to support namespace + local ruleinst = get_rule(target, rulename) + local script_group = root_group .. "/" .. ruleinst:fullname() if jobgraph:has(script_group) then table.insert(joborders, script_group) end -- cgit v1.3.1 From 77d90d253447ea6992f4f0a81d8df219e23b499c Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:42:30 +0800 Subject: improve target_cmds --- .../modules/private/action/build/build_binary.lua | 6 +- .../private/action/build/build_moduleonly.lua | 4 +- .../modules/private/action/build/build_object.lua | 4 +- .../modules/private/action/build/build_shared.lua | 4 +- .../modules/private/action/build/build_static.lua | 4 +- .../modules/private/action/build/link_objects.lua | 8 +- .../modules/private/action/build/prepare_files.lua | 4 +- xmake/modules/private/action/build/target.lua | 118 +++++++++++++++------ xmake/plugins/project/cmake/cmakelists.lua | 13 +-- xmake/plugins/project/utils/target_cmds.lua | 47 ++++++++ 10 files changed, 156 insertions(+), 56 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua index 72263e6c8..3863b8550 100644 --- a/xmake/modules/private/action/build/build_binary.lua +++ b/xmake/modules/private/action/build/build_binary.lua @@ -22,16 +22,16 @@ import("build_object") import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) +function main(jobgraph, target, opt) local objects_group = target:fullname() .. "/objects" local jobsize = jobgraph:size() jobgraph:group(objects_group, function () - build_object(jobgraph, target) + build_object(jobgraph, target, opt) end) if jobgraph:size() > jobsize then local link_group = target:fullname() .. "/link" jobgraph:group(link_group, function () - target_buildutils.add_linkjobs(jobgraph, target) + target_buildutils.add_linkjobs(jobgraph, target, opt) end) jobgraph:add_orders(objects_group, link_group) end diff --git a/xmake/modules/private/action/build/build_moduleonly.lua b/xmake/modules/private/action/build/build_moduleonly.lua index e3cc87c27..32c07375e 100644 --- a/xmake/modules/private/action/build/build_moduleonly.lua +++ b/xmake/modules/private/action/build/build_moduleonly.lua @@ -21,6 +21,6 @@ -- imports import("build_object") -function main(jobgraph, target) - build_object(jobgraph, target) +function main(jobgraph, target, opt) + build_object(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_object.lua b/xmake/modules/private/action/build/build_object.lua index 2a6baa59a..08f82092b 100644 --- a/xmake/modules/private/action/build/build_object.lua +++ b/xmake/modules/private/action/build/build_object.lua @@ -21,6 +21,6 @@ -- imports import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) - target_buildutils.add_filejobs(jobgraph, target, {job_kind = "build"}) +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_shared.lua b/xmake/modules/private/action/build/build_shared.lua index 077489710..9223093b4 100644 --- a/xmake/modules/private/action/build/build_shared.lua +++ b/xmake/modules/private/action/build/build_shared.lua @@ -21,6 +21,6 @@ -- imports import("build_binary") -function main(jobgraph, target) - build_binary(jobgraph, target) +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_static.lua b/xmake/modules/private/action/build/build_static.lua index 8b9b2ffaf..6d47394ed 100644 --- a/xmake/modules/private/action/build/build_static.lua +++ b/xmake/modules/private/action/build/build_static.lua @@ -21,6 +21,6 @@ -- imports import("build_binary") -function main(jobgraph, target) - build_binary(jobgraph, target) +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua index 25214d27e..396896d00 100644 --- a/xmake/modules/private/action/build/link_objects.lua +++ b/xmake/modules/private/action/build/link_objects.lua @@ -60,9 +60,13 @@ function _do_link_target(target, opt) values = depvalues, files = depfiles, dryrun = dryrun}) end -function main(jobgraph, target) +function main(jobgraph, target, opt) + opt = opt or {} + local buildcmds = opt.buildcmds local linkjob = target:fullname() .. "/link_objects" jobgraph:add(linkjob, function (index, total, opt) - _do_link_target(target, opt) + if not buildcmds then + _do_link_target(target, opt) + end end) end diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua index d8ca351ca..9f0f47ef0 100644 --- a/xmake/modules/private/action/build/prepare_files.lua +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -22,6 +22,6 @@ import("core.base.option") import("private.action.build.target", {alias = "target_buildutils"}) -function main(jobgraph, target) - target_buildutils.add_filejobs(jobgraph, target, {job_kind = "prepare"}) +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index beb6f9cb2..68f6bda53 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -119,14 +119,16 @@ function _add_targetjobs_orders(jobgraph, target, dep, opt) end -- add target jobs for the builtin script -function add_targetjobs_for_builtin_script(jobgraph, target, job_kind) +function add_targetjobs_for_builtin_script(jobgraph, target, opt) + opt = opt or {} + local job_kind = opt.job_kind if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then if job_kind == "prepare" then - import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target) + import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target, opt) elseif job_kind == "link" then - import("private.action.build.link_objects", {anonymous = true})(jobgraph, target) + import("private.action.build.link_objects", {anonymous = true})(jobgraph, target, opt) else - import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target) + import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target, opt) end end end @@ -135,6 +137,7 @@ end function add_targetjobs_for_script(jobgraph, target, instance, opt) opt = opt or {} local has_script = false + local buildcmds = opt.buildcmds local job_prefix = target:fullname() if target == instance then job_prefix = job_prefix .. "/target" @@ -143,7 +146,7 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) end -- call script - if not has_script then + if not has_script and not buildcmds then local script_name = opt.script_name local script = instance:script(script_name) if script then @@ -185,9 +188,14 @@ function add_targetjobs_for_script(jobgraph, target, instance, opt) if scriptcmd then local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + if buildcmds then + -- only generate cmds and do not run them, use cases: e.g. project generator + scriptcmd(target, buildcmds, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end end) has_script = true end @@ -220,7 +228,8 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) local has_script = false local script_opt = { script_name = script_name, - scriptcmd_name = scriptcmd_name + scriptcmd_name = scriptcmd_name, + buildcmds = opt.buildcmds } for _, instance in ipairs(instances) do -- we need to use this group to sort rule scripts with add_orders @@ -238,7 +247,7 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- call builtin script, e.g. on_prepare, on_build, ... if not has_script and stage == "" then - add_targetjobs_for_builtin_script(jobgraph, target, job_kind) + add_targetjobs_for_builtin_script(jobgraph, target, opt) end end) @@ -265,10 +274,15 @@ function add_targetjobs(jobgraph, target, opt) _g.pkgenvs = pkgenvs end + local buildcmds = opt.buildcmds local job_kind = opt.job_kind local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) local job_end = string.format("%s/end_%s", target:fullname(), job_kind) jobgraph:add(job_begin, function (index, total, opt) + if buildcmds then + return + end + -- enter package environments -- https://github.com/xmake-io/xmake/issues/4033 -- @@ -293,6 +307,10 @@ function add_targetjobs(jobgraph, target, opt) end) jobgraph:add(job_end, function (index, total, opt) + if buildcmds then + return + end + -- restore environments if target:pkgenvs() then pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() @@ -307,9 +325,17 @@ function add_targetjobs(jobgraph, target, opt) end) -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) end @@ -341,6 +367,7 @@ end function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) opt = opt or {} local has_script = false + local buildcmds = opt.buildcmds local job_prefix = target:fullname() local file_group = sourcebatch.rulename if target == instance then @@ -350,7 +377,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) end -- call script files - if not has_script then + if not has_script and not buildcmds then local script_files_name = opt.script_files_name local script_files = instance:script(script_files_name) if script_files then @@ -383,7 +410,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) end -- call script file - if not has_script then + if not has_script and not buildcmds then local script_file_name = opt.script_file_name local script_file = instance:script(script_file_name) if script_file then @@ -434,9 +461,14 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local distcc = instance:extraconf(scriptcmd_files_name, "distcc") local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + scriptcmd_files(target, buildcmds, sourcebatch, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end end) has_script = true end @@ -455,12 +487,20 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) local distcc = instance:extraconf(scriptcmd_file_name, "distcc") local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) jobgraph:add(jobname, function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, buildcmds, sourcefile, {sourcekind = sourcekind}) + end + else + local batchcmds_ = batchcmds.new({target = target}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) end) has_script = true end @@ -473,6 +513,7 @@ end -- function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} + local buildcmds = opt.buildcmds local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" @@ -523,7 +564,8 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) script_file_name = script_file_name, script_files_name = script_files_name, scriptcmd_file_name = scriptcmd_file_name, - scriptcmd_files_name = scriptcmd_files_name + scriptcmd_files_name = scriptcmd_files_name, + buildcmds = buildcmds } local has_target_script = false for _, instance in ipairs(instances) do @@ -575,9 +617,17 @@ function add_filejobs(jobgraph, target, opt) end -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files - local group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) - local group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) - local group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + end jobgraph:add_orders(group_before, group, group_after) end @@ -608,9 +658,17 @@ end function add_linkjobs(jobgraph, target, opt) opt = opt or {} opt.job_kind = "link" - local group = add_targetjobs_with_stage(jobgraph, target, "", opt) - local group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) - local group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end jobgraph:add_orders(group_before, group, group_after) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index f826cc453..9d2fc453a 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -29,7 +29,6 @@ import("core.project.rule") import("core.platform.platform") import("lib.detect.find_tool") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -1201,9 +1200,6 @@ end -- add target custom commands function _add_target_custom_commands(cmakelists, target, outputdir) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules if _can_native_support_for_cxxmodules() then @@ -1212,17 +1208,12 @@ function _add_target_custom_commands(cmakelists, target, outputdir) -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index debc6fdc5..c25a2c3e3 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -122,3 +122,50 @@ function prepare_targets() target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) end +-- get target buildcmds +function get_target_buildcmds(target, opt) + opt = opt or {} + local progress_wrapper = {} + progress_wrapper.current = function () + return count + end + progress_wrapper.total = function () + return total + end + progress_wrapper.percent = function () + if total and total > 0 then + return math.floor((count * 100) / total) + else + return 0 + end + end + debug.setmetatable(progress_wrapper, { + __tostring = function () + -- we do not output any progress info for the project generators + return "" + end + }) + local buildcmds = batchcmds.new({target = target}) + local jobgraph = target_buildutils.get_targetjobs({target}, { + job_kind = "build", + buildcmds = buildcmds, + with_stages = hashset.from(opt.stages or {}), + ignored_rules = hashset.from(opt.ignored_rules or {}), + progress = progress_wrapper}) + if jobgraph and not jobgraph:empty() then + local jobqueue = jobgraph:build() + while true do + local job = jobqueue:getfree() + if job then + if job.run then + job.run() + end + jobqueue:remove(job) + else + break + end + end + end + return buildcmds:cmds() +end + -- cgit v1.3.1 From 4109abcddf430a8fad253566b23db6f8c73b5366 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:44:19 +0800 Subject: ignore rules and stages --- xmake/modules/private/action/build/target.lua | 38 ++++++++++++++++----------- xmake/plugins/project/utils/target_cmds.lua | 8 +++--- 2 files changed, 28 insertions(+), 18 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index 68f6bda53..477dd3023 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -208,6 +208,7 @@ end function add_targetjobs_with_stage(jobgraph, target, stage, opt) opt = opt or {} local job_kind = opt.job_kind + local ignored_rules = opt.ignored_rules -- the group name, e.g. foo/after_prepare, bar/before_build local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) @@ -220,8 +221,11 @@ function add_targetjobs_with_stage(jobgraph, target, stage, opt) -- call target and rules script local instances = {target} - for _, r in ipairs(target:orderules()) do - table.insert(instances, r) + for _, ruleinst in ipairs(target:orderules()) do + -- we only ignore some builtin rules, so we need not to use fullname. + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + table.insert(instances, ruleinst) + end end local jobsize = jobgraph:size() jobgraph:group(group_name, function () @@ -491,7 +495,7 @@ function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) if buildcmds then local sourcekind = sourcebatch.sourcekind for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - scriptcmd_file(target, buildcmds, sourcefile, {sourcekind = sourcekind}) + scriptcmd_file(target, buildcmds, sourcefile, {progress = opt.progress, sourcekind = sourcekind}) end else local batchcmds_ = batchcmds.new({target = target}) @@ -514,6 +518,7 @@ end function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) opt = opt or {} local buildcmds = opt.buildcmds + local ignored_rules = opt.ignored_rules local job_kind = opt.job_kind local job_kind_file = job_kind .. "_file" local job_kind_files = job_kind .. "_files" @@ -538,20 +543,23 @@ function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) for _, sourcebatch in pairs(sourcebatches) do local rulename = sourcebatch.rulename if rulename then + -- we only ignore some builtin rules, so we need not to use fullname. local ruleinst = rule_utils.get_rule(target, rulename) - sourcebatches_map[ruleinst] = sourcebatch - -- avoid duplicate scripts being called twice in the target, - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - if ruleinst:script("build_file") or ruleinst:script("build_files") then - table.insert(sourcebatches_for_target, sourcebatch) + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + table.insert(instances, ruleinst) end - table.insert(instances, ruleinst) else table.insert(sourcebatches_for_target, sourcebatch) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index c25a2c3e3..835117d50 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -150,17 +150,19 @@ function get_target_buildcmds(target, opt) job_kind = "build", buildcmds = buildcmds, with_stages = hashset.from(opt.stages or {}), - ignored_rules = hashset.from(opt.ignored_rules or {}), - progress = progress_wrapper}) + ignored_rules = hashset.from(opt.ignored_rules or {})}) if jobgraph and not jobgraph:empty() then + local total = jobgraph:size() + local index = 0 local jobqueue = jobgraph:build() while true do local job = jobqueue:getfree() if job then if job.run then - job.run() + job.run(index, total, {progress = progress_wrapper}) end jobqueue:remove(job) + index = index + 1 else break end -- cgit v1.3.1 From 95e69eeea88e1ff30f67ef94bab7ba1652b026a4 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:46:21 +0800 Subject: fix link jobs --- xmake/modules/private/action/build/target.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua index 477dd3023..c302408ac 100644 --- a/xmake/modules/private/action/build/target.lua +++ b/xmake/modules/private/action/build/target.lua @@ -664,7 +664,7 @@ end -- add link jobs for the given target function add_linkjobs(jobgraph, target, opt) - opt = opt or {} + opt = table.clone(opt or {}) opt.job_kind = "link" local with_stages = opt.with_stages local group, group_before, group_after -- cgit v1.3.1 From 699e40f3e45469b3597f5fa24ad6bb2c922b8a4b Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 9 Apr 2025 23:48:29 +0800 Subject: move rule_group to deprecated --- xmake/actions/build/deprecated/kinds/object.lua | 2 +- .../actions/build/deprecated/kinds/rule_groups.lua | 95 ++++++++++++++++++++++ xmake/modules/private/utils/rule_groups.lua | 95 ---------------------- xmake/plugins/project/clang/compile_commands.lua | 14 +--- xmake/plugins/project/utils/target_cmds.lua | 90 -------------------- xmake/plugins/project/vstudio/impl/vs201x.lua | 17 ++-- 6 files changed, 103 insertions(+), 210 deletions(-) create mode 100644 xmake/actions/build/deprecated/kinds/rule_groups.lua delete mode 100644 xmake/modules/private/utils/rule_groups.lua (limited to 'xmake/modules') diff --git a/xmake/actions/build/deprecated/kinds/object.lua b/xmake/actions/build/deprecated/kinds/object.lua index b5359f79e..c4ed8721a 100644 --- a/xmake/actions/build/deprecated/kinds/object.lua +++ b/xmake/actions/build/deprecated/kinds/object.lua @@ -25,7 +25,7 @@ import("core.project.config") import("core.project.project") import("async.runjobs") import("private.utils.batchcmds") -import("private.utils.rule_groups") +import("rule_groups") -- has scripts for the custom rule function _has_scripts_for_rule(ruleinst, suffix) diff --git a/xmake/actions/build/deprecated/kinds/rule_groups.lua b/xmake/actions/build/deprecated/kinds/rule_groups.lua new file mode 100644 index 000000000..d64a75d62 --- /dev/null +++ b/xmake/actions/build/deprecated/kinds/rule_groups.lua @@ -0,0 +1,95 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file rule_groups.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") + +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + +-- get max depth of rule +function _get_rule_max_depth(target, ruleinst, depth) + local max_depth = depth + for _, depname in ipairs(ruleinst:get("deps")) do + local dep = get_rule(target, depname) + local dep_depth = depth + if ruleinst:extraconf("deps", depname, "order") then + dep_depth = dep_depth + 1 + end + local cur_depth = _get_rule_max_depth(target, dep, dep_depth) + if cur_depth > max_depth then + max_depth = cur_depth + end + end + return max_depth +end + +-- build sourcebatch groups for target +function _build_sourcebatch_groups_for_target(groups, target, sourcebatches) + local group = groups[1] + for _, sourcebatch in pairs(sourcebatches) do + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local item = group[rulename] or {} + item.target = target + item.sourcebatch = sourcebatch + group[rulename] = item + end +end + +-- build sourcebatch groups for rules +function _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) + for _, sourcebatch in pairs(sourcebatches) do + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local ruleinst = get_rule(target, rulename) + local depth = _get_rule_max_depth(target, ruleinst, 1) + local group = groups[depth] + if group == nil then + group = {} + groups[depth] = group + end + local item = group[rulename] or {} + item.rule = ruleinst + item.sourcebatch = sourcebatch + group[rulename] = item + end +end + +-- build sourcebatch groups by rule dependencies order, e.g. `add_deps("qt.ui", {order = true})` +-- +-- @see https://github.com/xmake-io/xmake/issues/2814 +-- +function build_sourcebatch_groups(target, sourcebatches) + local groups = {{}} + _build_sourcebatch_groups_for_target(groups, target, sourcebatches) + _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) + if #groups > 0 then + groups = table.reverse(groups) + end + return groups +end + diff --git a/xmake/modules/private/utils/rule_groups.lua b/xmake/modules/private/utils/rule_groups.lua deleted file mode 100644 index d64a75d62..000000000 --- a/xmake/modules/private/utils/rule_groups.lua +++ /dev/null @@ -1,95 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file rule_groups.lua --- - --- imports -import("core.base.option") -import("core.project.rule") -import("core.project.config") -import("core.project.project") - --- get rule --- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target -function get_rule(target, rulename) - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - return ruleinst -end - --- get max depth of rule -function _get_rule_max_depth(target, ruleinst, depth) - local max_depth = depth - for _, depname in ipairs(ruleinst:get("deps")) do - local dep = get_rule(target, depname) - local dep_depth = depth - if ruleinst:extraconf("deps", depname, "order") then - dep_depth = dep_depth + 1 - end - local cur_depth = _get_rule_max_depth(target, dep, dep_depth) - if cur_depth > max_depth then - max_depth = cur_depth - end - end - return max_depth -end - --- build sourcebatch groups for target -function _build_sourcebatch_groups_for_target(groups, target, sourcebatches) - local group = groups[1] - for _, sourcebatch in pairs(sourcebatches) do - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local item = group[rulename] or {} - item.target = target - item.sourcebatch = sourcebatch - group[rulename] = item - end -end - --- build sourcebatch groups for rules -function _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) - for _, sourcebatch in pairs(sourcebatches) do - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = get_rule(target, rulename) - local depth = _get_rule_max_depth(target, ruleinst, 1) - local group = groups[depth] - if group == nil then - group = {} - groups[depth] = group - end - local item = group[rulename] or {} - item.rule = ruleinst - item.sourcebatch = sourcebatch - group[rulename] = item - end -end - --- build sourcebatch groups by rule dependencies order, e.g. `add_deps("qt.ui", {order = true})` --- --- @see https://github.com/xmake-io/xmake/issues/2814 --- -function build_sourcebatch_groups(target, sourcebatches) - local groups = {{}} - _build_sourcebatch_groups_for_target(groups, target, sourcebatches) - _build_sourcebatch_groups_for_rules(groups, target, sourcebatches) - if #groups > 0 then - groups = table.reverse(groups) - end - return groups -end - diff --git a/xmake/plugins/project/clang/compile_commands.lua b/xmake/plugins/project/clang/compile_commands.lua index c36cb8ade..4cf6eb1ab 100644 --- a/xmake/plugins/project/clang/compile_commands.lua +++ b/xmake/plugins/project/clang/compile_commands.lua @@ -27,7 +27,6 @@ import("core.project.project") import("core.language.language") import("private.utils.batchcmds") import("private.utils.executable_path") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("actions.test.main", {rootdir = os.programdir(), alias = "test_action"}) @@ -242,25 +241,16 @@ end -- add target commands function _add_target_commands(jsonfile, target) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before"}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups) + local cmds_before = target_cmds.get_target_buildcmds(target, {stages = {"before", "on"}}) _add_target_custom_commands(jsonfile, target, "before", cmds_before) -- add target source commands _add_target_source_commands(jsonfile, target) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after"}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after"}) + local cmds_after = target_cmds.get_target_buildcmds(target, {stages = {"after"}}) _add_target_custom_commands(jsonfile, target, "after", cmds_after) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index 835117d50..b18acb943 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -24,98 +24,8 @@ import("core.project.config") import("core.base.hashset") import("core.project.rule") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("private.action.build.target", {alias = "target_buildutils"}) --- this sourcebatch is built? -function _sourcebatch_is_built(sourcebatch) - -- we can only use rulename to filter them because sourcekind may be bound to multiple rules - local rulename = sourcebatch.rulename - if rulename == "c.build" or rulename == "c++.build" - or rulename == "asm.build" or rulename == "cuda.build" - or rulename == "objc.build" or rulename == "objc++.build" - or rulename == "win.sdk.resource" then - return true - end -end - --- get target buildcmd commands -function get_target_buildcmd(target, cmds, opt) - opt = opt or {} - local suffix = opt.suffix - local ignored_rules = hashset.from(opt.ignored_rules or {}) - for _, ruleinst in ipairs(target:orderules()) do - if not ignored_rules:has(ruleinst:name()) then - local scriptname = "buildcmd" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - end - end -end - --- get target buildcmd_files commands -function get_target_buildcmd_files(target, cmds, sourcebatch, opt) - opt = opt or {} - - -- get rule - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - local ignored_rules = hashset.from(opt.ignored_rules or {}) - if ignored_rules:has(ruleinst:name()) then - return - end - - -- generate commands for xx_buildcmd_files - local suffix = opt.suffix - local scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcebatch, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - - -- generate commands for xx_buildcmd_file - if not script then - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcefile, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - end - end -end - --- get target buildcmd commands of source group -function get_target_buildcmd_sourcegroups(target, cmds, sourcegroups, opt) - for idx, group in irpairs(sourcegroups) do - for _, item in pairs(group) do - -- buildcmd scripts are always in rule, so we need to ignore target item (item.target). - local sourcebatch = item.sourcebatch - if item.rule then - if not _sourcebatch_is_built(sourcebatch) then - get_target_buildcmd_files(target, cmds, sourcebatch, opt) - end - end - end - end -end - -- prepare targets function prepare_targets() local targets_root = target_buildutils.get_root_targets() diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index 0a23a1847..e3b9ddc95 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -39,7 +39,6 @@ import("private.action.require.install", {alias = "install_requires"}) import("private.action.run.runenvs") import("actions.config.configfiles", {alias = "generate_configfiles", rootdir = os.programdir()}) import("private.utils.batchcmds") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) function _translate_path(dir, vcxprojdir) @@ -132,24 +131,18 @@ function _make_custom_commands(target, vcxprojdir) return _translate_path(p, vcxprojdir) end) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules = _get_cxxmodules_rules() -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + -- rule.on_buildcmd_files should also be executed before building the target + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) local commands = {} for _, cmd in ipairs(cmds_before) do -- cgit v1.3.1 From cf75a18fc72cf469a3cfc9aa5ff550deb2b29c27 Mon Sep 17 00:00:00 2001 From: Doekin Date: Thu, 10 Apr 2025 09:23:48 +0800 Subject: improve `emscripten` detection by including `emcc.py` search --- xmake/modules/detect/sdks/find_emsdk.lua | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'xmake/modules') diff --git a/xmake/modules/detect/sdks/find_emsdk.lua b/xmake/modules/detect/sdks/find_emsdk.lua index cbc6998b2..ec8553692 100644 --- a/xmake/modules/detect/sdks/find_emsdk.lua +++ b/xmake/modules/detect/sdks/find_emsdk.lua @@ -33,10 +33,17 @@ function _find_emsdkdir(sdkdir) table.insert(paths, sdkdir) end table.insert(paths, "$(env EMSDK)") + if is_host("linux") then + table.join2(paths, {"/usr/share/emscripten/", "/usr/lib/emscripten/"}) + end local emsdk = find_file("emsdk.py", paths, {suffixes = subdirs}) if emsdk then return path.directory(emsdk) end + local emcc_py = find_file("emcc.py", paths, {suffixes = subdirs}) + if emcc_py then + return path.directory(emcc_py) + end end -- find emsdk @@ -53,6 +60,7 @@ function _find_emsdk(sdkdir) local subdirs = {} table.insert(subdirs, path.join("*", "emscripten")) local emcc = find_file("emcc", sdkdir, {suffixes = subdirs}) + emcc = emcc or find_file("emcc.py", sdkdir) if emcc then emscripten = path.directory(emcc) end -- cgit v1.3.1 From 45ca6fea545babb73773b536e6e056e6ca71fe4f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:32:21 +0800 Subject: suppress warnings --- xmake/modules/private/action/require/impl/package.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules') diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 87547ab3f..02fac7a3e 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -222,7 +222,7 @@ function _load_require(require_str, requires_extra, opt) -- check require options local extra_options = hashset.of("plat", "arch", "kind", "host", "targetos", "alias", "group", "system", "option", "default", "optional", "debug", - "verify", "external", "private", "build", "configs", "version") + "verify", "external", "private", "build", "configs", "version", "public") for name, value in pairs(require_extra) do if not extra_options:has(name) then wprint("add_requires(\"%s\") has unknown option: {%s=%s}!", require_str, name, tostring(value)) -- cgit v1.3.1 From 1268cad62854df59e71757a55fddd74ebde6c65f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:41:42 +0800 Subject: improve cmake:find_package #6296 --- xmake/modules/package/manager/cmake/find_package.lua | 2 ++ 1 file changed, 2 insertions(+) (limited to 'xmake/modules') diff --git a/xmake/modules/package/manager/cmake/find_package.lua b/xmake/modules/package/manager/cmake/find_package.lua index 6c3a9f194..4b1c390f0 100644 --- a/xmake/modules/package/manager/cmake/find_package.lua +++ b/xmake/modules/package/manager/cmake/find_package.lua @@ -81,6 +81,8 @@ function _find_package(cmake, name, opt) if moduledirs then for _, moduledir in ipairs(moduledirs) do cmakefile:print("list(APPEND CMAKE_MODULE_PATH \"%s\")", (moduledir:gsub("\\", "/"))) + -- https://github.com/xmake-io/xmake/issues/6296 + cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (moduledir:gsub("\\", "/"))) end end -- e.g. set(Boost_USE_STATIC_LIB ON) -- cgit v1.3.1 From fa122559e2fcb560f7be7b6f61241b8caf524c65 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 10 Apr 2025 22:42:03 +0800 Subject: improve cmake:find_package again #6296 --- xmake/modules/package/manager/cmake/find_package.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'xmake/modules') diff --git a/xmake/modules/package/manager/cmake/find_package.lua b/xmake/modules/package/manager/cmake/find_package.lua index 4b1c390f0..4c71291d7 100644 --- a/xmake/modules/package/manager/cmake/find_package.lua +++ b/xmake/modules/package/manager/cmake/find_package.lua @@ -81,10 +81,16 @@ function _find_package(cmake, name, opt) if moduledirs then for _, moduledir in ipairs(moduledirs) do cmakefile:print("list(APPEND CMAKE_MODULE_PATH \"%s\")", (moduledir:gsub("\\", "/"))) - -- https://github.com/xmake-io/xmake/issues/6296 - cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (moduledir:gsub("\\", "/"))) end end + -- https://github.com/xmake-io/xmake/issues/6296 + local prefixdirs = configs.prefixdirs or opt.prefixdirs + if prefixdirs then + for _, prefixdir in ipairs(prefixdirs) do + cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (prefixdir:gsub("\\", "/"))) + end + end + -- e.g. set(Boost_USE_STATIC_LIB ON) local presets = configs.presets or opt.presets if presets then -- cgit v1.3.1 From 389765de963238987ebe27b8237554bdffb7bfdc Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Thu, 10 Apr 2025 13:05:11 +0800 Subject: package(autoconf): resolve issue with backslash-separated `CXX` path in MSYS --- xmake/modules/package/tools/autoconf.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'xmake/modules') diff --git a/xmake/modules/package/tools/autoconf.lua b/xmake/modules/package/tools/autoconf.lua index 8d312a76a..821ab759f 100644 --- a/xmake/modules/package/tools/autoconf.lua +++ b/xmake/modules/package/tools/autoconf.lua @@ -434,6 +434,7 @@ function buildenvs(package, opt) if is_host("windows") then envs.CC = _translate_windows_bin_path(envs.CC) + envs.CXX = _translate_windows_bin_path(envs.CXX) envs.AS = _translate_windows_bin_path(envs.AS) envs.AR = _translate_windows_bin_path(envs.AR) envs.LD = _translate_windows_bin_path(envs.LD) -- cgit v1.3.1