From a278d3bd3e74027e7fd1f878c7d8c991caf1a75f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 20 Mar 2025 23:16:51 +0800 Subject: improve to find cycle --- .../modules/modules_support/dependency_scanner.lua | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 9d552d0fd..df8c85441 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -222,7 +222,7 @@ function _generate_dependencies(target, sourcebatch, opt) local changed = false if opt.batchjobs then local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) + runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed end, {comax = jobs, total = #sourcebatch.sourcefiles}) @@ -415,19 +415,20 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, objectfile in ipairs(cycle) do - local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, objectfile in ipairs(cycle) do + local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + table.insert(names, name or cppfile) + end + local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) table.insert(names, name or cppfile) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) - table.insert(names, name or cppfile) - raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - - local objectfiles_sorted = table.reverse(dag:topological_sort()) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then @@ -465,7 +466,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) end end end - if insert then + if insert then table.insert(build_objectfiles, objectfile) table.insert(link_objectfiles, objectfile) elseif external and not external.from_moduleonly then -- cgit v1.3.1 From 741da62196bcb64c88e386f1a769462211c63b4e Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 21 Mar 2025 23:03:29 +0800 Subject: add queue and improve graph --- tests/modules/graph/test.lua | 8 +- tests/modules/queue/test.lua | 35 ++++ xmake/core/base/graph.lua | 216 +++++++++++++++++++-- xmake/core/base/queue.lua | 125 ++++++++++++ .../sandbox/modules/import/core/base/queue.lua | 22 +++ xmake/core/tool/builder.lua | 2 +- xmake/modules/async/jobgraph.lua | 54 ++---- xmake/modules/cli/amalgamate.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 3 +- 9 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 tests/modules/queue/test.lua create mode 100644 xmake/core/base/queue.lua create mode 100644 xmake/core/sandbox/modules/import/core/base/queue.lua (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 4bbefe069..33e1f3fa7 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -1,6 +1,6 @@ import("core.base.graph") -function test_topological_sort(t) +function test_topo_sort(t) local edges = { {0, 5}, {0, 2}, @@ -18,7 +18,7 @@ function test_topological_sort(t) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local order_path = dag:topological_sort() + local order_path = dag:topo_sort() local orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -28,7 +28,7 @@ function test_topological_sort(t) end dag = dag:reverse() - order_path = dag:topological_sort() + order_path = dag:topo_sort() orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -53,7 +53,7 @@ function test_find_cycle(t) local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) - local _, has_cycle = dag:topological_sort() + local _, has_cycle = dag:topo_sort() t:require(has_cycle) end diff --git a/tests/modules/queue/test.lua b/tests/modules/queue/test.lua new file mode 100644 index 000000000..b577e04c9 --- /dev/null +++ b/tests/modules/queue/test.lua @@ -0,0 +1,35 @@ +import("core.base.queue") + +function test_push(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + t:are_equal(d:first(), 1) + t:are_equal(d:last(), 5) + local idx = 1 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + +function test_pop(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + d:pop() + t:are_equal(d:first(), 2) + t:are_equal(d:last(), 5) + local idx = 2 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index ffcd815c5..448eebb88 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -19,9 +19,10 @@ -- -- load modules -local table = require("base/table") -local list = require("base/list") -local object = require("base/object") +local table = require("base/table") +local queue = require("base/queue") +local object = require("base/object") +local hashset = require("base/hashset") -- define module local graph = graph or object { _init = {"_directed"} } {true} @@ -58,6 +59,9 @@ function graph:clear() self._edges = {} self._adjacent_edges = {} self._edges_map = {} + + -- clear partial topological sort state + self:partial_topo_sort_reset() end -- is empty? @@ -116,10 +120,171 @@ function graph:remove_vertex(v) end end end + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() + end +end + +-- check if there's a cycle in the remaining unprocessed nodes +function graph:_check_cycle_in_remaining() + -- if all remaining nodes have in-degree > 0, we have a cycle + if self._topo_remaining_count > 0 and self._topo_remaining_count == self._topo_non_zero_indegree_count then + self._topo_has_cycle = true + return true end + return false end --- topological sort, use Kahn's algorithm +-- reset partial topological sort state +function graph:partial_topo_sort_reset() + self._topo_in_progress = false + self._topo_in_degree = nil + self._topo_queue = nil + self._topo_processed = nil + self._topo_has_cycle = nil + self._topo_remaining_count = nil + self._topo_non_zero_indegree_count = nil +end + +-- get next batch of nodes in topological order with limit +-- +-- @param limit the maximum number of nodes to return +-- @return array of nodes with zero in-degree, empty when complete +-- @return has_cycle indicates if a cycle was detected +-- +-- e.g. +-- +-- add_edge(a, b) -- a depend on b +-- add_edge(b, c) -- b depend on c +-- +-- local batch1, has_cycle = g:partial_topo_sort_next(1) -- returns {c} +-- local batch2, has_cycle = g:partial_topo_sort_next(1) -- returns {b} +-- local batch3, has_cycle = g:partial_topo_sort_next(1) -- returns {a} +-- local batch4, has_cycle = g:partial_topo_sort_next(1) -- returns {} (empty, all done) +-- +function graph:partial_topo_sort_next(limit) + if not self:is_directed() then + return {}, false + end + + limit = limit or math.huge + + -- check if we already detected a cycle + if self._topo_has_cycle then + return {}, true + end + + -- initialize topological sort state if not already in progress + if not self._topo_in_progress then + -- calculate in-degree for each vertex + self._topo_in_degree = {} + for _, v in ipairs(self:vertices()) do + self._topo_in_degree[v] = 0 + end + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = (self._topo_in_degree[w] or 0) + 1 + end + end + end + end + + -- initialize queue with vertices that have no incoming edges + self._topo_queue = queue.new() + for _, v in ipairs(self:vertices()) do + if self._topo_in_degree[v] == 0 then + self._topo_queue:push(v) + end + end + + -- track processed vertices + self._topo_processed = hashset.new() + self._topo_in_progress = true + + -- track counts for efficient cycle detection + self._topo_remaining_count = #self:vertices() + self._topo_non_zero_indegree_count = self._topo_remaining_count - self._topo_queue:size() + + -- quick cycle detection: if no nodes have zero in-degree, we have a cycle + if self._topo_queue:empty() and self._topo_remaining_count > 0 then + self._topo_has_cycle = true + return {}, true + end + end + + -- return empty batch if queue is empty (all processed or cycle detected) + if self._topo_queue:empty() then + -- check if all vertices were processed + local processed_count = self._topo_processed:size() + self._topo_has_cycle = processed_count ~= #self:vertices() + + -- if this is the first call and we detect a cycle, mark as complete + if processed_count == 0 then + self._topo_in_progress = false + end + + return {}, self._topo_has_cycle + end + + -- collect up to 'limit' nodes with zero in-degree + local batch = {} + while not self._topo_queue:empty() and #batch < limit do + local v = self._topo_queue:pop() + table.insert(batch, v) + self._topo_processed:insert(v) + self._topo_remaining_count = self._topo_remaining_count - 1 + end + + -- update in-degrees based on the nodes in this batch + for _, v in ipairs(batch) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + self._topo_in_degree[w] = self._topo_in_degree[w] - 1 + + -- update non-zero in-degree count + if self._topo_in_degree[w] == 0 then + self._topo_non_zero_indegree_count = self._topo_non_zero_indegree_count - 1 + + -- if in-degree becomes zero, add to queue for next batch + if not self._topo_processed:has(w) then + self._topo_queue:push(w) + end + end + end + end + end + end + + -- early cycle detection - if all remaining nodes have in-degree > 0 + if self:_check_cycle_in_remaining() then + return batch, true + end + + -- if queue is now empty and all vertices processed, reset state + if self._topo_queue:empty() then + local processed_count = self._topo_processed:size() + if processed_count == #self:vertices() then + self._topo_in_progress = false + else + -- if queue is empty but we still have unprocessed nodes, we have a cycle + self._topo_has_cycle = true + end + end + + return batch, self._topo_has_cycle +end + +-- topological sort, use kahn's algorithm -- -- e.g. -- @@ -127,8 +292,35 @@ end -- add_edge(b, c) -- b depend on c -- -- it will return {c, b, a} -function graph:topological_sort(opt) - opt = opt or {} +--[[ +function graph:topo_sort() + if not self:is_directed() then + return + end + + -- reset partial sort state to ensure we start fresh + self:partial_topo_sort_reset() + + local order_vertices = {} + local batch_size = math.huge -- no limit, get all at once + + -- get all nodes in one go + local batch, has_cycle = self:partial_topo_sort_next(batch_size) + while #batch > 0 do + for _, v in ipairs(batch) do + table.insert(order_vertices, v) + end + batch, has_cycle = self:partial_topo_sort_next(batch_size) + + -- quick exit if cycle is detected + if has_cycle then + break + end + end + + return order_vertices, has_cycle +end]] +function graph:topo_sort() if not self:is_directed() then return end @@ -153,10 +345,10 @@ function graph:topological_sort(opt) end -- queue of vertices with no incoming edges (no dependencies) - local queue = list.new() + local queue = queue.new() for _, v in ipairs(self:vertices()) do if in_degree[v] == 0 then - queue:insert(v) + queue:push(v) end end @@ -166,7 +358,7 @@ function graph:topological_sort(opt) -- process queue while not queue:empty() do -- remove a vertex with no incoming edges - local v = queue:remove_first() + local v = queue:pop() table.insert(order_vertices, v) -- for each outgoing edge, remove it and update in-degrees @@ -178,7 +370,7 @@ function graph:topological_sort(opt) in_degree[w] = in_degree[w] - 1 -- if in-degree becomes zero, add to queue if in_degree[w] == 0 then - queue:insert(w) + queue:push(w) end end end @@ -263,6 +455,9 @@ function graph:add_edge(from, to) edges_map[to][from] = true end table.insert(self._edges, e) + + -- reset partial topological sort state since graph structure changed + self:partial_topo_sort_reset() end -- has the given edge? @@ -340,4 +535,3 @@ end -- return module: graph return graph - diff --git a/xmake/core/base/queue.lua b/xmake/core/base/queue.lua new file mode 100644 index 000000000..dbdc1d87a --- /dev/null +++ b/xmake/core/base/queue.lua @@ -0,0 +1,125 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- load modules +local object = require("base/object") + +-- define module +local queue = queue or object {_init = {"_first", "_last"}} {1, 0} + +-- clear queue +function queue:clear() + self._first = 1 + self._last = 0 +end + +-- push item to queue +function queue:push(item) + local last = self._last + 1 + self._last = last + self[last] = item +end + +-- pop item from queue +function queue:pop() + local first = self._first + if first > self._last then + return nil + end + + local value = self[first] + self[first] = nil + self._first = first + 1 + return value +end + +-- get queue size +function queue:size() + return self._last - self._first + 1 +end + +-- is queue empty? +function queue:empty() + return self._first > self._last +end + +-- peek the first item of queue +function queue:first() + if self._first > self._last then + return nil + end + return self[self._first] +end + +-- peek the last item of queue +function queue:last() + if self._first > self._last then + return nil + end + return self[self._last] +end + +-- iterator for all items (forward) +-- +-- e.g. +-- +-- for item in queue:items() do +-- print(item) +-- end +-- +function queue:items() + local index = self._first - 1 + local last = self._last + return function() + index = index + 1 + if index <= last then + return self[index] + end + end +end + +-- iterator for all items (reverse) +function queue:ritems() + local index = self._last + 1 + local first = self._first + return function() + index = index - 1 + if index >= first then + return self[index] + end + end +end + +-- clone queue +function queue:clone() + local q = queue.new() + for i = self._first, self._last do + q:push(self[i]) + end + return q +end + +-- new queue +function queue.new() + return queue() +end + +-- return module: queue +return queue diff --git a/xmake/core/sandbox/modules/import/core/base/queue.lua b/xmake/core/sandbox/modules/import/core/base/queue.lua new file mode 100644 index 000000000..aab0a8214 --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/base/queue.lua @@ -0,0 +1,22 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- return module +return require("base/queue") diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index b0f802237..0e85911eb 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -669,7 +669,7 @@ function builder:_sort_links_of_items(items, opt) end if not gh:empty() then local has_cycle - links, has_cycle = gh:topological_sort() + links, has_cycle = gh:topo_sort() if has_cycle then local cycle = gh:find_cycle() if cycle then diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua index ca8725b28..4a0dac110 100644 --- a/xmake/modules/async/jobgraph.lua +++ b/xmake/modules/async/jobgraph.lua @@ -25,18 +25,20 @@ import("core.base.graph") import("core.base.hashset") -- define module -local jobqueue = jobqueue or object {_init = {"_jobgraph", "_queue"}} +local jobqueue = jobqueue or object {_init = {"_jobgraph"}} local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_dirty"}} --- build the job queue -function jobqueue:_build() +-- remove the given job from the job queue +function jobqueue:remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() local graph = self._jobgraph local dag = graph._dag - local queue = self._queue -- build job queue - queue:clear() - local order_jobs, has_cycle = dag:topological_sort() + local order_jobs, has_cycle = dag:partial_topo_sort_next(1) if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -48,42 +50,8 @@ function jobqueue:_build() raise("%s: circular job dependency detected!\n%s", graph, table.concat(names, "\n -> ")) end end - for _, job in ipairs(order_jobs) do - print("insert", job.name) - queue:insert(job) - end -end - --- update the job queue -function jobqueue:_update() - local graph = self._jobgraph - if graph._dirty then - self:_build() - graph._dirty = false - end -end - --- remove the given job from the job queue -function jobqueue:remove(job) - local queue = self._queue - print("remove", job.name) - queue:remove(job) - -- TODO remove deps -end - --- get a free job from the job queue -function jobqueue:getfree() - self:_update() - - local queue = self._queue - if queue:empty() then - return - end - - -- TODO - for job in queue:ritems() do - print("get free job", job.name) - return job + if order_jobs then + return table.unwrap(order_jobs) end end @@ -150,7 +118,7 @@ end -- build a job queue function jobgraph:build() - return jobqueue {self, list.new()} + return jobqueue {self} end -- get jobs diff --git a/xmake/modules/cli/amalgamate.lua b/xmake/modules/cli/amalgamate.lua index 8c7ec6fd7..d0a5ba40c 100644 --- a/xmake/modules/cli/amalgamate.lua +++ b/xmake/modules/cli/amalgamate.lua @@ -96,7 +96,7 @@ function _generate_file(target, inputpaths, outputpath, uniqueid) _generate_include_graph(target, inputpaths, gh, {}) -- sort file paths and remove root path - local filepaths = gh:topological_sort() + local filepaths = gh:topo_sort() table.remove(filepaths, 1) -- generate amalgamate file diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index df8c85441..94c14e60d 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -415,7 +415,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local objectfiles_sorted, has_cycle = dag:topological_sort({reverse = true}) + local objectfiles_sorted, has_cycle = dag:topo_sort() if has_cycle then local cycle = dag:find_cycle() if cycle then @@ -429,6 +429,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end end + objectfiles_sorted = table.reverse(objectfiles_sorted) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then -- cgit v1.3.1 From 1b735b328bda4d207675a1406dfe714e934de0c8 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 22:50:16 +0800 Subject: improve c++modules for jobgraph --- xmake/actions/build/target_utils.lua | 4 +- .../rules/c++/modules/modules_support/builder.lua | 83 ++++++++++++++-- .../c++/modules/modules_support/clang/builder.lua | 104 +++++++++++++++++++- .../modules/modules_support/dependency_scanner.lua | 2 +- .../c++/modules/modules_support/gcc/builder.lua | 107 ++++++++++++++++++++- .../c++/modules/modules_support/msvc/builder.lua | 101 ++++++++++++++++++- xmake/rules/c++/modules/xmake.lua | 24 +++-- 7 files changed, 398 insertions(+), 27 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/actions/build/target_utils.lua b/xmake/actions/build/target_utils.lua index 6eddda91f..798009ab6 100644 --- a/xmake/actions/build/target_utils.lua +++ b/xmake/actions/build/target_utils.lua @@ -292,7 +292,7 @@ end -- get target jobs function get_targetjobs(targets_root, opt) - local jobgraph = async_jobgraph.new() + local jobgraph = async_jobgraph.new(opt.job_kind) local targetrefs = {} for _, target in ipairs(targets_root) do add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) @@ -542,7 +542,7 @@ end -- get files jobs function get_filejobs(targets_root, opt) - local jobgraph = async_jobgraph.new() + local jobgraph = async_jobgraph.new(opt.job_kind) local targetrefs = {} for _, target in ipairs(targets_root) do add_filejobs_and_deps(jobgraph, target, targetrefs, opt) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a71e2c392..0139bc107 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -34,6 +34,7 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles + local jobgraph = opt.jobgraph for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then @@ -45,7 +46,8 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + table.insert(deps, depname) end opt.build_module(deps, module, name, objectfile, cppfile) @@ -204,7 +206,6 @@ end -- "file": "foo.cppm" -- } function _generate_meta_module_info(target, name, sourcefile, requires) - local modulehash = compiler_support.get_modulehash(target, sourcefile) local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} @@ -266,7 +267,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "_populate_module_map" + local populate_jobname = target:name() .. "/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -278,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = name and target:name() .. name or cppfile + local job_name = target:fullname() .. "/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -288,9 +289,39 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) end +-- build modules for jobgraph +function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local jobdeps = {} + local build_modules_group = target:fullname() .. "/build_modules" + jobgraph:group(build_modules_group, function () + + -- add populate module job + local populate_jobname = target:fullname() .. "/populate_module_map" + jobgraph:add(populate_jobname, function(index, total, opt) + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + end) + + -- add module jobs + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, module, name, objectfile, cppfile) + local jobname = target:fullname() .. "/" .. (name or cppfile) + _builder(target).make_module_jobgraph(target, jobgraph, { + module = module, objectfile = objectfile, cppfile = cppfile + }) + jobdeps[jobname] = table.join(populate_jobname, deps) + end}) + ) + end) + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end +end + -- build modules for batchcmds function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local depmtime = 0 opt.progress = opt.progress or 0 @@ -307,7 +338,7 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op batchcmds:set_depmtime(depmtime) end --- generate headerunits for batchjobs +-- build headerunits for batchjobs function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -345,7 +376,43 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules end end --- generate headerunits for batchcmds +-- build headerunits for jobgraph +function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + local build_modules_group = target:fullname() .. "/build_modules" + local build_headerunits_group = target:fullname() .. "/build_headerunits" + jobgraph:group(build_headerunits_group, function () + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:fullname() .. "/" .. key + _builder(target).make_headerunit_buildjobs(target, + job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + end + })) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end + end) + jobgraph:add_orders(build_headerunits_group, build_modules_group) +end + +-- build headerunits for batchcmds function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -391,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 972694254..fa2d46844 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:name() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -278,6 +278,75 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local is_mapped_bmi = mapped_bmi ~= nil + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -351,6 +420,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show(jobopt.progress, + "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", + target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 94c14e60d..e576bd562 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -220,7 +220,7 @@ end -- generate dependency files function _generate_dependencies(target, sourcebatch, opt) local changed = false - if opt.batchjobs then + if opt.jobgraph then local jobs = option.get("jobs") or os.default_njob() runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index a0a43db5a..20cda94f8 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -241,6 +241,69 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + end + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + if sourcefile then + _compile(target, flags, sourcefile, opt.objectfile) + end + os.tryrm(module_mapper) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -337,6 +400,46 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + if option.get("diagnosis") then + print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _compile(target, + _make_headerunitflags(target, headerunit, headerunit_mapper, opt), + path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + + + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 5998db62f..cb2dcad1d 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -257,16 +257,15 @@ end -- build module file for batchjobs function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then @@ -326,6 +325,71 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -401,6 +465,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f967dd133..30d65cf2a 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -71,7 +71,7 @@ rule("c++.build.modules.builder") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") -- parallel build support to accelerate `xmake build` to build modules - before_build_files(function(target, batchjobs, sourcebatch, opt) + before_build_files(function(target, jobgraph, sourcebatch, opt) if target:data("cxx.has_modules") then import("modules_support.compiler_support") import("modules_support.dependency_scanner") @@ -102,7 +102,7 @@ rule("c++.build.modules.builder") end opt = opt or {} - opt.batchjobs = true + opt.jobgraph = true compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) @@ -112,11 +112,17 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build modules - builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - - -- build headerunits and we need to do it before building modules - builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + if jobgraph.add_orders then + -- build modules + builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + else + -- build modules, deprecated + builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + end sourcebatch.objectfiles = link_objectfiles else @@ -129,7 +135,7 @@ rule("c++.build.modules.builder") -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end - end, {batch = true}) + end, {jobgraph = true, batch = true}) -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) @@ -163,7 +169,7 @@ rule("c++.build.modules.builder") end opt = opt or {} - opt.batchjobs = false + opt.jobgraph = false compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) -- cgit v1.3.1 From 8b0ba84af0611cc61e9398c579ab55e9087320d4 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 22:51:19 +0800 Subject: use target:fullname() --- .../rules/c++/modules/modules_support/builder.lua | 34 +++++++++++----------- .../c++/modules/modules_support/clang/builder.lua | 28 +++++++++--------- .../modules_support/clang/dependency_scanner.lua | 2 +- .../modules/modules_support/compiler_support.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 8 ++--- .../c++/modules/modules_support/gcc/builder.lua | 28 +++++++++--------- .../modules_support/gcc/dependency_scanner.lua | 2 +- .../c++/modules/modules_support/msvc/builder.lua | 28 +++++++++--------- .../modules_support/msvc/dependency_scanner.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 6 ++-- 10 files changed, 70 insertions(+), 70 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 0139bc107..59f00b64f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -130,7 +130,7 @@ function _try_reuse_modules(target, modules) end local mapped = get_from_target_mapper(dep, name) if mapped then - compiler_support.memcache():set2(target:name() .. name, "reuse", true) + compiler_support.memcache():set2(target:fullname() .. name, "reuse", true) add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) break end @@ -158,8 +158,8 @@ function should_build(target, sourcefile, bmifile, opt) for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) if m then - local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) - or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key) + local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) + or compiler_support.memcache():get2("should_build_in_" .. target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -174,7 +174,7 @@ function should_build(target, sourcefile, bmifile, opt) if opt.name then local m = get_from_target_mapper(target, opt.name) if m and m.opt and m.opt.target then - local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -224,7 +224,7 @@ end function _target_module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end function _is_duplicated_headerunit(target, key) @@ -252,7 +252,7 @@ function _builder(target) end function mark_build(target, name) - compiler_support.memcache():set2("should_build_in_" .. target:name(), name, true) + compiler_support.memcache():set2("should_build_in_" .. target:fullname(), name, true) end -- build batchjobs for modules @@ -263,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -349,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:name() .. key + local job_name = target:fullname() .. "/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -462,7 +462,7 @@ function generate_metadata(target, modules) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) - progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name) local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) json.savefile(metafilepath, metadata) end, {comax = jobs, total = #public_modules}) @@ -471,20 +471,20 @@ end -- flush target module mapper keys function flush_target_module_mapper_keys(target) local memcache = compiler_support.memcache() - memcache:set2(target:name(), "module_mapper_keys", nil) + memcache:set2(target:fullname(), "module_mapper_keys", nil) end -- get or create a target module mapper function get_target_module_mapper(target) local memcache = compiler_support.memcache() - local mapper = memcache:get2(target:name(), "module_mapper") + local mapper = memcache:get2(target:fullname(), "module_mapper") if not mapper then mapper = {} - memcache:set2(target:name(), "module_mapper", mapper) + memcache:set2(target:fullname(), "module_mapper", mapper) end -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit - local mapper_keys = memcache:get2(target:name(), "module_mapper_keys") + local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys") if not mapper_keys then mapper_keys = {} for _, item in pairs(mapper) do @@ -492,7 +492,7 @@ function get_target_module_mapper(target) mapper_keys[item.key] = item end end - memcache:set2(target:name(), "module_mapper_keys", mapper_keys) + memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys) end return mapper, mapper_keys end @@ -530,7 +530,7 @@ end -- check if dependencies changed function is_dependencies_changed(target, module) - local cachekey = target:name() .. module.name + local cachekey = target:fullname() .. module.name local requires = hashset.from(table.keys(module.requires or {})) local oldrequires = compiler_support.memcache():get2(cachekey, "oldrequires") local changed = false diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index fa2d46844..dd9a6fad5 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -134,7 +134,7 @@ function _get_requiresflags(target, module, opt) local modulefileflag = compiler_support.get_modulefileflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -216,11 +216,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -263,11 +263,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) end else @@ -288,7 +288,7 @@ function make_module_jobgraph(target, jobgraph, opt) local jobname = target:fullname() .. "/" .. (name or opt.cppfile) jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -331,11 +331,11 @@ function make_module_jobgraph(target, jobgraph, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) end else @@ -354,7 +354,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -374,11 +374,11 @@ function make_module_buildcmds(target, batchcmds, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds, is_mapped_bmi = is_mapped_bmi}) end else @@ -409,7 +409,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local depvalues = {compinst:program(), compflags} if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end @@ -440,7 +440,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil if opt.build then progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:name(), headerunit.name) + target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end @@ -458,7 +458,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 06b522ce8..715e8a07d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -36,7 +36,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index cf64a08bb..a5a32e2c5 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -224,7 +224,7 @@ function modules_cachedir(target, opt) end function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:name() + local key = path.directory(modulepath) .. target:fullname() return hash.uuid(key):split("-", {plain = true})[1]:lower() end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index e576bd562..4a8f08909 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -222,7 +222,7 @@ function _generate_dependencies(target, sourcebatch, opt) local changed = false if opt.jobgraph then local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) + runjobs(target:fullname() .. "/module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed end, {comax = jobs, total = #sourcebatch.sourcefiles}) @@ -235,7 +235,7 @@ function _generate_dependencies(target, sourcebatch, opt) end -- get module dependencies function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:name() .. "/" .. sourcebatch.rulename + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename local modules = compiler_support.memcache():get2("modules", cachekey) if modules == nil then modules = compiler_support.localcache():get2("modules", cachekey) @@ -476,8 +476,8 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) objectfiles_sorted_set:remove(objectfile) if name ~= "std" and name ~= "std.compat" then culleds = culleds or {} - culleds[target:name()] = culleds[target:name()] or {} - table.insert(culleds[target:name()], format("%s -> %s", name, cppfile)) + culleds[target:fullname()] = culleds[target:fullname()] or {} + table.insert(culleds[target:fullname()], format("%s -> %s", name, cppfile)) end end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 20cda94f8..38a9e3d04 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -76,7 +76,7 @@ end function _module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end -- generate a module mapper file for build a headerunit @@ -135,7 +135,7 @@ end -- function _generate_modulemapper_file(target, module, cppfile) local maplines = _get_maplines(target, module) - local mapper_path = path.join(os.tmpdir(), target:name():replace(" ", "_"), name or cppfile:replace(" ", "_")) + local mapper_path = path.join(os.tmpdir(), target:fullname():replace(" ", "_"), name or cppfile:replace(" ", "_")) local mapper_content = {} table.insert(mapper_content, "root " .. path.unix(os.projectdir())) for _, mapline in ipairs(maplines) do @@ -186,7 +186,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) sourcefile = opt.cppfile, job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -217,13 +217,13 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -249,7 +249,7 @@ function make_module_jobgraph(target, jobgraph, opt) jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -280,13 +280,13 @@ function make_module_jobgraph(target, jobgraph, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -312,7 +312,7 @@ function make_module_buildcmds(target, batchcmds, opt) local module_mapperflag = compiler_support.get_modulemapperflag(target) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -335,13 +335,13 @@ function make_module_buildcmds(target, batchcmds, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -383,7 +383,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end @@ -421,7 +421,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end @@ -452,7 +452,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) if option.get("diagnosis") then batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) end diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 11dab5669..e087c6d4e 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index cb2dcad1d..fe1f25bb4 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -167,7 +167,7 @@ function _get_requiresflags(target, module, opt) local headerunitflag = compiler_support.get_headerunitflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -175,7 +175,7 @@ function _get_requiresflags(target, module, opt) local deps_flags = {} for required in requires:orderitems() do local dep_module = get_from_target_mapper(target, required) - assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) + assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:fullname()) local mapflag local bmifile = dep_module.bmi @@ -268,7 +268,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -310,11 +310,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else @@ -333,7 +333,7 @@ function make_module_jobgraph(target, jobgraph, opt) jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -375,11 +375,11 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else @@ -397,7 +397,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -417,11 +417,11 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, provide, {batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) end else @@ -454,7 +454,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local name = headerunit.unique and headerunit.name or headerunit.path if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end @@ -485,7 +485,7 @@ function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifil local name = headerunit.unique and headerunit.name or headerunit.path if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end @@ -503,7 +503,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), target:objectfile(headerunit.path)) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index 491a27cbf..044510fab 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) local changed = false depend.on_changed(function () - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) local outputdir = compiler_support.get_outputdir(target, sourcefile) local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 30d65cf2a..f4b7f744e 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -129,7 +129,7 @@ rule("c++.build.modules.builder") sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs @@ -191,7 +191,7 @@ rule("c++.build.modules.builder") sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else sourcebatch.sourcefiles = {} @@ -228,7 +228,7 @@ rule("c++.build.modules.install") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets if compiler_support.contains_modules(target) then - local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + local modules = compiler_support.localcache():get2(target:fullname(), "c++.modules") builder.generate_metadata(target, modules) compiler_support.add_installfiles_for_modules(target) -- cgit v1.3.1 From c0867ffd08915f2f80decbf97da17b685fb74525 Mon Sep 17 00:00:00 2001 From: ruki Date: Mon, 31 Mar 2025 23:05:27 +0800 Subject: fix build modules --- xmake/modules/private/action/build/object.lua | 2 +- .../rules/c++/modules/modules_support/builder.lua | 26 +++++++++++----------- .../c++/modules/modules_support/clang/builder.lua | 6 ++--- .../c++/modules/modules_support/gcc/builder.lua | 7 +++--- .../c++/modules/modules_support/msvc/builder.lua | 7 +++--- xmake/rules/c++/modules/xmake.lua | 11 +++------ 6 files changed, 28 insertions(+), 31 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 5118dbb82..0028b4b6f 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -165,7 +165,7 @@ function _add_jobgraph(target, jobgraph, sourcebatch, opt) local objectfile = sourcebatch.objectfiles[i] local dependfile = sourcebatch.dependfiles[i] local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) - local jobname = target:fullname() .. "/" .. sourcefile + local jobname = target:fullname() .. "/obj/" .. sourcefile jobgraph:add(jobname, function (index, total, jobopt) local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) build_object(target, sourcefile, build_opt) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 59f00b64f..0fd9920b1 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -46,7 +46,7 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep table.insert(deps, depname) end @@ -263,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -279,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/" .. (name or cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -292,11 +292,11 @@ end -- build modules for jobgraph function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) local jobdeps = {} - local build_modules_group = target:fullname() .. "/build_modules" + local build_modules_group = target:fullname() .. "/module/build_modules" jobgraph:group(build_modules_group, function () -- add populate module job - local populate_jobname = target:fullname() .. "/populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" jobgraph:add(populate_jobname, function(index, total, opt) _try_reuse_modules(target, modules) _builder(target).populate_module_map(target, modules) @@ -305,7 +305,7 @@ function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local jobname = target:fullname() .. "/" .. (name or cppfile) + local jobname = target:fullname() .. "/module/" .. (name or cppfile) _builder(target).make_module_jobgraph(target, jobgraph, { module = module, objectfile = objectfile, cppfile = cppfile }) @@ -349,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -385,14 +385,14 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local build_modules_group = target:fullname() .. "/build_modules" - local build_headerunits_group = target:fullname() .. "/build_headerunits" + local build_modules_group = target:fullname() .. "/module/build_modules" + local build_headerunits_group = target:fullname() .. "/module/build_headerunits" jobgraph:group(build_headerunits_group, function () local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/" .. key + local job_name = target:fullname() .. "/module/" .. key _builder(target).make_headerunit_buildjobs(target, job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) end @@ -458,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index dd9a6fad5..ec6692c35 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -285,7 +285,7 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 38a9e3d04..05e8380da 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -247,7 +247,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index fe1f25bb4..8f582438b 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -263,9 +263,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:fullname() .. "/populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then @@ -331,7 +331,8 @@ function make_module_jobgraph(target, jobgraph, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f4b7f744e..42eab0c19 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -112,15 +112,12 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles + -- build modules and headerunits, and we need to build headerunits first if jobgraph.add_orders then - -- build modules builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - else - -- build modules, deprecated + else -- deprecated builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) end @@ -179,10 +176,8 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build headerunits + -- build headerunits and modules builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - - -- build modules builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles -- cgit v1.3.1 From 8f6a3130eff588abda27b22fa86e936b6500f50c Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 5 Apr 2025 23:30:36 +0800 Subject: fix headerunit --- .../rules/c++/modules/modules_support/builder.lua | 49 ++++++++++++++++------ .../c++/modules/modules_support/gcc/builder.lua | 2 - xmake/rules/c++/modules/xmake.lua | 11 +---- 3 files changed, 38 insertions(+), 24 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 0fd9920b1..7d5ebd0c8 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -45,11 +45,15 @@ function _build_modules(target, sourcebatch, modules, opt) cppfile = cppfile or module.cppfile local deps = {} - for _, dep in ipairs(table.keys(module.requires or {})) do + for name, req in pairs(module.requires or {}) do + -- we need to use the full path as dep name if requre item is headerunit + local dep = name + if req.method:startswith("include-") and req.path then + dep = path.normalize(req.path) + end local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep table.insert(deps, depname) end - opt.build_module(deps, module, name, objectfile, cppfile) ::continue:: @@ -58,7 +62,6 @@ end -- build target headerunits function _build_headerunits(target, headerunits, opt) - local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true}) if opt.stl_headerunit then outputdir = path.join(outputdir, "stl") @@ -72,11 +75,9 @@ function _build_headerunits(target, headerunits, opt) local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) local key = path.normalize(headerunit.path) local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) - if build then mark_build(target, key) end - opt.build_headerunit(headerunit, key, bmifile, outputdir, build) end end @@ -292,6 +293,7 @@ end -- build modules for jobgraph function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) local jobdeps = {} + local jobsize = jobgraph:size() local build_modules_group = target:fullname() .. "/module/build_modules" jobgraph:group(build_modules_group, function () @@ -313,10 +315,8 @@ function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) end}) ) end) - for jobname, deps in pairs(jobdeps) do - for _, depname in ipairs(deps) do - jobgraph:add_orders(depname, jobname) - end + if jobgraph:size() > jobsize then + return build_modules_group, jobdeps end end @@ -385,7 +385,7 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local build_modules_group = target:fullname() .. "/module/build_modules" + local jobsize = jobgraph:size() local build_headerunits_group = target:fullname() .. "/module/build_headerunits" jobgraph:group(build_headerunits_group, function () local build_headerunits = function(headerunits) @@ -393,7 +393,7 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) local job_name = target:fullname() .. "/module/" .. key - _builder(target).make_headerunit_buildjobs(target, + _builder(target).make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) end })) @@ -409,12 +409,13 @@ function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, build_headerunits(user_headerunits) end end) - jobgraph:add_orders(build_headerunits_group, build_modules_group) + if jobgraph:size() > jobsize then + return build_headerunits_group + end end -- build headerunits for batchcmds function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) if not user_headerunits and not stl_headerunits then return @@ -441,6 +442,28 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end +-- build modules and headerunits, and we need to build headerunits first +function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) + if jobgraph.add_orders then + local build_modules_group, jobdeps = build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local build_headerunits_group = build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + if build_modules_group then + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end + if build_headerunits_group then + jobgraph:add_orders(build_headerunits_group, build_modules_group) + end + end + else -- deprecated + build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + end +end + +-- generate metadata function generate_metadata(target, modules) local public_modules for _, module in table.orderpairs(modules) do diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 05e8380da..4033eefd5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -361,7 +361,6 @@ end -- build headerunit file for batchjobs function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local _headerunit = headerunit _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) @@ -443,7 +442,6 @@ end -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 42eab0c19..5f17e2099 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -112,15 +112,8 @@ rule("c++.build.modules.builder") local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build modules and headerunits, and we need to build headerunits first - if jobgraph.add_orders then - builder.build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - builder.build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - else -- deprecated - builder.build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - builder.build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - end - + -- build modules and headerunits + builder.build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else sourcebatch.objectfiles = {} -- cgit v1.3.1 From 99f68b5c58d822170c84e2b6c0b3f7f974e009be Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 20:58:25 +0800 Subject: remove unused prepare jobs --- .../rules/c++/modules/modules_support/builder.lua | 46 +++++++++++- .../modules/modules_support/compiler_support.lua | 14 ---- .../modules/modules_support/dependency_scanner.lua | 60 ++++++++-------- xmake/rules/c++/modules/xmake.lua | 84 ++++++---------------- 4 files changed, 96 insertions(+), 108 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 7d5ebd0c8..2a0977524 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -457,7 +457,10 @@ function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, o jobgraph:add_orders(build_headerunits_group, build_modules_group) end end - else -- deprecated + elseif jobgraph.runcmds then + build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + elseif jobgraph.newjob then -- deprecated build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) end @@ -571,3 +574,44 @@ function is_dependencies_changed(target, module) end return requires, changed end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch, opt) + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + end + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) + end + end + + -- patch objectfiles and dependencies + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + + local dependfile = target:dependfile(sourcefile or objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index a5a32e2c5..eed30a320 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -69,20 +69,6 @@ function strip_flags(target, flags) return _compiler_support(target).strip_flags(target, flags) end --- patch sourcebatch -function patch_sourcebatch(target, sourcebatch) - sourcebatch.sourcekind = "cxx" - sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - table.insert(sourcebatch.objectfiles, objectfile) - - local dependfile = target:dependfile(sourcefile or objectfile) - table.insert(sourcebatch.dependfiles, dependfile) - end -end - -- get bmi extension function get_bmi_extension(target) return _compiler_support(target).get_bmi_extension() diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 4a8f08909..94b8a4804 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -202,9 +202,8 @@ function _get_edges(nodes, modules) return edges end -function _get_package_modules(target, package, opt) +function _get_package_modules(target, package) local package_modules - local modulesdir = path.join(package:installdir(), "modules") local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) for _, metafile in ipairs(metafiles) do @@ -213,42 +212,39 @@ function _get_package_modules(target, package, opt) local moduleonly = not package:libraryfiles() package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata, external = {moduleonly = moduleonly}} end - return package_modules end --- generate dependency files -function _generate_dependencies(target, sourcebatch, opt) - local changed = false - if opt.jobgraph then - local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/module_dependency_scanner", function(index) - local sourcefile = sourcebatch.sourcefiles[index] - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end, {comax = jobs, total = #sourcebatch.sourcefiles}) - else - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end - end - return changed -end --- get module dependencies -function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:fullname() .. "/" .. sourcebatch.rulename - local modules = compiler_support.memcache():get2("modules", cachekey) - if modules == nil then - modules = compiler_support.localcache():get2("modules", cachekey) - opt.progress = opt.progress or 0 - local changed = _generate_dependencies(target, sourcebatch, opt) - if changed or modules == nil then +-- generate module dependencies +function generate_module_dependencies(target, jobgraph, sourcebatch, opt) + local parsejob = target:fullname() .. "/parse_module_dependencies" + jobgraph:add(parsejob, function (index, total, opt) + local changed = compiler_support.memcache():get2("modules", "dependencies_changed") + if changed then + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) - modules = _parse_dependencies_data(target, moduleinfos) + local modules = _parse_dependencies_data(target, moduleinfos) compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() end - compiler_support.memcache():set2("modules", cachekey, modules) + end) + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local jobname = target:fullname() .. "/generate_module_dependencies/" .. sourcefile + jobgraph:add(jobname, function (index, total, opt) + local changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) + if changed then + compiler_support.memcache():set2("modules", "dependencies_changed", true) + end + end) + jobgraph:add_orders(jobname, parsejob) end +end + +-- get module dependencies +function get_module_dependencies(target, sourcebatch) + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename + local modules = compiler_support.localcache():get2("modules", cachekey) + assert(modules, "no module dependencies!") return modules end @@ -387,7 +383,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end -- extract packages modules dependencies -function get_all_packages_modules(target, opt) +function get_all_packages_modules(target) -- parse all meta-info and append their informations to the package store local packages = target:pkgs() or {} @@ -397,7 +393,7 @@ function get_all_packages_modules(target, opt) local packages_modules for _, package in table.orderpairs(packages) do - local package_modules = _get_package_modules(target, package, opt) + local package_modules = _get_package_modules(target, package) if package_modules then packages_modules = packages_modules or {} table.join2(packages_modules, package_modules) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 5f17e2099..87429f90c 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -70,6 +70,20 @@ rule("c++.build.modules.builder") set_sourcekinds("cxx") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") + -- generate module dependencies + on_prepare_files(function (target, jobgraph, sourcebatch, opt) + if target:data("cxx.has_modules") then + import("modules_support.builder") + import("modules_support.dependency_scanner") + + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + + -- generate module dependencies + dependency_scanner.generate_module_dependencies(target, jobgraph, sourcebatch, opt) + end + end, {jobgraph = true}) + -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, jobgraph, sourcebatch, opt) if target:data("cxx.has_modules") then @@ -77,36 +91,11 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end - - opt = opt or {} - opt.jobgraph = true - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) @@ -134,45 +123,18 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end - - opt = opt or {} - opt.jobgraph = false - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles -- build headerunits and modules - builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - + builder.build_modules_and_headerunits(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else -- avoid duplicate linking of object files of non-module programs -- cgit v1.3.1 From da540028213b73ca014bc7aa0f7938a12e344ee8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 6 Apr 2025 21:09:49 +0800 Subject: fix generate deps --- xmake/rules/c++/modules/modules_support/builder.lua | 20 ++++++++++---------- xmake/rules/c++/modules/xmake.lua | 6 ------ 2 files changed, 10 insertions(+), 16 deletions(-) (limited to 'xmake/rules/c++/modules/modules_support') diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 2a0977524..7ec120cc9 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -34,7 +34,6 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles - local jobgraph = opt.jobgraph for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then @@ -51,7 +50,7 @@ function _build_modules(target, sourcebatch, modules, opt) if req.method:startswith("include-") and req.path then dep = path.normalize(req.path) end - local depname = jobgraph and (target:fullname() .. "/module/" .. dep) or dep + local depname = target:fullname() .. "/module/" .. dep table.insert(deps, depname) end opt.build_module(deps, module, name, objectfile, cppfile) @@ -279,11 +278,11 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/module/" .. (name or cppfile) - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, - {module = module, objectfile = objectfile, cppfile = cppfile}) - end + build_module = function(deps, module, name, objectfile, cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, + {module = module, objectfile = objectfile, cppfile = cppfile}) + end })) -- build batchjobs for modules @@ -330,9 +329,10 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, module, _, objectfile, cppfile) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) - end + build_module = function(_, module, _, objectfile, cppfile) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, { + module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + end })) batchcmds:set_depmtime(depmtime) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 87429f90c..d0c0e0387 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -91,9 +91,6 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- patch sourcebatch - builder.patch_sourcebatch(target, sourcebatch) - -- get module dependencies local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then @@ -123,9 +120,6 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- patch sourcebatch - builder.patch_sourcebatch(target, sourcebatch) - -- get module dependencies local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- cgit v1.3.1