diff options
| author | Christian Rendina <[email protected]> | 2025-04-10 09:50:37 +0200 |
|---|---|---|
| committer | Christian Rendina <[email protected]> | 2025-04-10 09:50:37 +0200 |
| commit | 78723913d76fb8b615df34541236b8ea588d30db (patch) | |
| tree | b6b6ff550e4a2f73d94c63a61876cec31a4b3ae3 | |
| parent | 2051c13f735a626f7b6fd8b98aa69f01fd9cef7e (diff) | |
| parent | fd49b7754c6709a87b5beb5526788bbc1a663965 (diff) | |
Merge branch 'dev' of https://github.com/xmake-io/xmake into dev
103 files changed, 3569 insertions, 1272 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 915d92307..a2266582b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## master (unreleased) +### Changes + +* [#6202](https://github.com/xmake-io/xmake/issues/6202): Improve rule API and build dependency order +* [#5624](https://github.com/xmake-io/xmake/discussions/5624): Enable auto build when calling xmake run by default +* [#5526](https://github.com/xmake-io/xmake/discussions/5526): Use MD/MDd runtimes for msvc by default +* [#5545](https://github.com/xmake-io/xmake/discussions/5545): Use ninja generator for cmake package by default + ## v2.9.9 ### New features @@ -1968,6 +1975,13 @@ ## master (开发中) +### 改进 + +* [#6202](https://github.com/xmake-io/xmake/issues/6202): 改进 rule API 和构建顺序支持,提供统一 jobgraph 调度 +* [#5624](https://github.com/xmake-io/xmake/discussions/5624): `xmake run` 运行默认自动构建 +* [#5526](https://github.com/xmake-io/xmake/discussions/5526): msvc 默认切换到 MD/MDd 运行时 +* [#5545](https://github.com/xmake-io/xmake/discussions/5545): 构建 cmake 包,默认使用 Ninja 生成器 + ## v2.9.9 ### 新特性 diff --git a/tests/apis/rules_inject_deps/xmake.lua b/tests/apis/rules_inject_deps/xmake.lua index cf73449bb..880bcde32 100644 --- a/tests/apis/rules_inject_deps/xmake.lua +++ b/tests/apis/rules_inject_deps/xmake.lua @@ -1,10 +1,6 @@ rule("cppfront") set_extensions(".cpp2") - on_load(function (target) - local rule = target:rule("c++.build"):clone() - rule:add("deps", "cppfront", {order = true}) - target:rule_add(rule) - end) + add_orders("cppfront", "c++.build") on_build_file(function (target, sourcefile, opt) print("build cppfront file") local objectfile = target:objectfile(sourcefile:gsub("cpp2", "cpp")) diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua index ea167dca9..e05dc7bd2 100644 --- a/tests/apis/rules_order/xmake.lua +++ b/tests/apis/rules_order/xmake.lua @@ -1,7 +1,15 @@ rule("markdown") - add_deps("man", {order = true}) set_extensions(".md", ".markdown") + add_orders("man", "markdown") + + before_build(function (target) + print("before_build: markdown") + end) + after_build(function (target) + print("after_build: markdown") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -14,6 +22,14 @@ rule("markdown") rule("man") set_extensions(".man") + + before_build(function (target) + print("before_build: man") + end) + after_build(function (target) + print("after_build: man") + end) + before_build_file(function (target, sourcefile) print("before_build_file: %s", sourcefile) end) @@ -26,7 +42,7 @@ rule("man") target("test") set_kind("binary") - add_rules("markdown") + add_rules("markdown", "man") add_files("src/*.c") add_files("src/*.md") add_files("src/*.man") diff --git a/tests/modules/async/run_callback.lua b/tests/modules/async/run_callback.lua new file mode 100644 index 000000000..21e75014b --- /dev/null +++ b/tests/modules/async/run_callback.lua @@ -0,0 +1,19 @@ +import("core.base.scheduler") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test callback ====================================") + local t = os.mclock() + runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/async/run_jobgraph.lua b/tests/modules/async/run_jobgraph.lua new file mode 100644 index 000000000..fe53a54c6 --- /dev/null +++ b/tests/modules/async/run_jobgraph.lua @@ -0,0 +1,53 @@ +import("core.base.scheduler") +import("async.jobgraph") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function _test_basic() + print("==================================== test basic ====================================") + local jobs = jobgraph.new() + jobs:add("job/root", _jobfunc) + for i = 1, 3 do + jobs:add("job/" .. i, _jobfunc) + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + jobs:add_orders("job/" .. i .. "/" .. j, "job/" .. i, "job/root") + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + +function _test_group() + print("==================================== test group ====================================") + local jobs = jobgraph.new() + jobs:add("job/root", _jobfunc) + for i = 1, 3 do + jobs:add("job/" .. i, _jobfunc, {groups = "bar"}) + jobs:group("foo", function () + for j = 1, 50 do + jobs:add("job/" .. i .. "/" .. j, _jobfunc) + end + end) + end + jobs:add_orders("foo", "bar", "job/root") + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + +function main() + _test_basic() + _test_group() +end + diff --git a/tests/modules/async/run_jobpool.lua b/tests/modules/async/run_jobpool.lua new file mode 100644 index 000000000..5de54e36c --- /dev/null +++ b/tests/modules/async/run_jobpool.lua @@ -0,0 +1,28 @@ +import("core.base.scheduler") +import("private.async.jobpool") +import("async.runjobs") + +function _jobfunc(index, total, opt) + print("%s: run job (%d/%d)", scheduler.co_running(), index, total) + local dt = os.mclock() + os.sleep(1000) + dt = os.mclock() - dt + print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) +end + +function main() + print("==================================== test jobpool ====================================") + local jobs = jobpool.new() + local root = jobs:addjob("job/root", _jobfunc) + for i = 1, 3 do + local job = jobs:addjob("job/" .. i, _jobfunc, {rootjob = root}) + for j = 1, 50 do + jobs:addjob("job/" .. i .. "/" .. j, _jobfunc, {rootjob = job}) + end + end + t = os.mclock() + runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) + print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) + end}) +end + diff --git a/tests/modules/graph/test.lua b/tests/modules/graph/test.lua index 63095be12..fe1d63e8d 100644 --- a/tests/modules/graph/test.lua +++ b/tests/modules/graph/test.lua @@ -1,6 +1,6 @@ import("core.base.graph") -function test_topological_sort(t) +function test_topo_sort(t) local edges = { {0, 5}, {0, 2}, @@ -18,7 +18,7 @@ function test_topological_sort(t) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local order_path = dag:topological_sort() + local order_path = dag:topo_sort() local orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -28,7 +28,7 @@ function test_topological_sort(t) end dag = dag:reverse() - order_path = dag:topological_sort() + order_path = dag:topo_sort() orders = {} for i, v in ipairs(order_path) do orders[v] = i @@ -38,6 +38,138 @@ function test_topological_sort(t) end end +function test_paritail_topo_sort(t) + local function partiail_topo_sort(dag) + dag:partial_topo_sort_reset() + + local node, has_cycle + local order_vertices = {} + while true do + node, has_cycle = dag:partial_topo_sort_next() + if node then + table.insert(order_vertices, node) + dag:partial_topo_sort_remove(node) + else + if has_cycle then + raise("has cycle!") + end + break + end + end + + return order_vertices, has_cycle + end + + local edges = { + {0, 5}, + {0, 2}, + {0, 1}, + {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + {6, 4}, + {6, 0}, + {3, 2}, + {1, 4}, + {2, 9}, + } + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) + end + local order_path = partiail_topo_sort(dag) + local orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + for _, e in ipairs(edges) do + t:require(orders[e[1]] < orders[e[2]]) + end + + dag = dag:reverse() + order_path = partiail_topo_sort(dag) + orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + for _, e in ipairs(edges) do + t:require(orders[e[1]] > orders[e[2]]) + end +end + +function test_paritail_topo_sort_dynamic(t) + local function partiail_topo_sort(dag) + dag:partial_topo_sort_reset() + + local node, has_cycle + local order_vertices = {} + local dynamic_adjust = false + while true do + node, has_cycle = dag:partial_topo_sort_next() + if node then + if not dynamic_adjust then + dag:add_edge(1, 4) + dag:remove_vertex(6) + end + table.insert(order_vertices, node) + dag:partial_topo_sort_remove(node) + if not dynamic_adjust then + dag:add_edge(2, 9) + dynamic_adjust = true + end + else + if has_cycle then + raise("has cycle!") + end + break + end + end + + assert(#order_vertices == #dag:vertices(), "vertices count not matched, %d != %d", #order_vertices, #dag:vertices()) + return order_vertices, has_cycle + end + + local edges = { + {0, 5}, + {0, 2}, + {0, 1}, + {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + {6, 4}, + {6, 0}, + {3, 2}, + } + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) + end + local order_path = partiail_topo_sort(dag) + local orders = {} + for i, v in ipairs(order_path) do + orders[v] = i + end + edges = { + {0, 5}, + {0, 2}, + {0, 1}, + -- {3, 6}, + {3, 5}, + {3, 4}, + {5, 4}, + -- {6, 4}, + -- {6, 0}, + {3, 2}, + {1, 4}, + {2, 9} + } + for _, e in ipairs(edges) do + t:require(orders[e[1]] < orders[e[2]]) + end +end + function test_find_cycle(t) local edges = { {9, 1}, @@ -52,5 +184,8 @@ function test_find_cycle(t) end local cycle = dag:find_cycle() t:are_equal(cycle, {1, 6, 0}) + + local _, has_cycle = dag:topo_sort() + t:require(has_cycle) end diff --git a/tests/modules/queue/test.lua b/tests/modules/queue/test.lua new file mode 100644 index 000000000..b577e04c9 --- /dev/null +++ b/tests/modules/queue/test.lua @@ -0,0 +1,35 @@ +import("core.base.queue") + +function test_push(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + t:are_equal(d:first(), 1) + t:are_equal(d:last(), 5) + local idx = 1 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + +function test_pop(t) + local d = queue.new() + d:push(1) + d:push(2) + d:push(3) + d:push(4) + d:push(5) + d:pop() + t:are_equal(d:first(), 2) + t:are_equal(d:last(), 5) + local idx = 2 + for item in d:items() do + t:are_equal(item, idx) + idx = idx + 1 + end +end + diff --git a/tests/modules/scheduler/runjobs.lua b/tests/modules/scheduler/runjobs.lua deleted file mode 100644 index 4e2a98cc0..000000000 --- a/tests/modules/scheduler/runjobs.lua +++ /dev/null @@ -1,43 +0,0 @@ -import("core.base.scheduler") -import("private.async.jobpool") -import("async.runjobs") - -function _jobfunc(index, total, opt) - print("%s: run job (%d/%d)", scheduler.co_running(), index, total) - local dt = os.mclock() - os.sleep(1000) - dt = os.mclock() - dt - print("%s: run job (%d/%d) end, progress: %s, dt: %d ms", scheduler.co_running(), index, total, opt.progress, dt) -end - -function main() - - -- test callback - print("==================================== test callback ====================================") - local t = os.mclock() - runjobs("test", _jobfunc, {total = 100, comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) - - -- test jobs - print("==================================== test jobs ====================================") - local jobs = jobpool.new() - local root = jobs:addjob("job/root", function (index, total, opt) - _jobfunc(index, total, opt) - end) - for i = 1, 3 do - local job = jobs:addjob("job/" .. i, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = root}) - for j = 1, 50 do - jobs:addjob("job/" .. i .. "/" .. j, function (index, total, opt) - _jobfunc(index, total, opt) - end, {rootjob = job}) - end - end - t = os.mclock() - runjobs("test", jobs, {comax = 6, timeout = 1000, timer = function (running_jobs_indices) - print("%s: timeout (%d ms), running: %s", scheduler.co_running(), os.mclock() - t, table.concat(running_jobs_indices, ",")) - end}) -end - diff --git a/tests/projects/c++/modules/hello_with_pch/src/hello.mpp b/tests/projects/c++/modules/hello_with_pch/src/hello.mpp new file mode 100644 index 000000000..124bd72bc --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/hello.mpp @@ -0,0 +1,10 @@ +module; +#include <cstdio> + +export module hello; + +export namespace hello { + void say(const char* str) { + printf("%s\n", str); + } +} diff --git a/tests/projects/c++/modules/hello_with_pch/src/main.cpp b/tests/projects/c++/modules/hello_with_pch/src/main.cpp new file mode 100644 index 000000000..739309dd4 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/main.cpp @@ -0,0 +1,8 @@ +#include "test.h" + +import hello; + +int main() { + hello::say("hello module!"); + return 0; +} diff --git a/tests/projects/c++/modules/hello_with_pch/src/test.h b/tests/projects/c++/modules/hello_with_pch/src/test.h new file mode 100644 index 000000000..d5a732287 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/src/test.h @@ -0,0 +1 @@ +#include <string> diff --git a/tests/projects/c++/modules/hello_with_pch/xmake.lua b/tests/projects/c++/modules/hello_with_pch/xmake.lua new file mode 100644 index 000000000..79ddb6b58 --- /dev/null +++ b/tests/projects/c++/modules/hello_with_pch/xmake.lua @@ -0,0 +1,7 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +target("hello") + set_kind("binary") + set_pcxxheader("src/test.h") + add_files("src/*.cpp", "src/*.mpp") diff --git a/tests/projects/c++/modules/test_pch.lua b/tests/projects/c++/modules/test_pch.lua new file mode 100644 index 000000000..98286c229 --- /dev/null +++ b/tests/projects/c++/modules/test_pch.lua @@ -0,0 +1,31 @@ +import("lib.detect.find_tool") +import("core.base.semver") +import("detect.sdks.find_vstudio") +import("utils.ci.is_running", {alias = "ci_is_running"}) + +function _build() + if ci_is_running() then + os.run("xmake -rvD") + else + os.run("xmake -r") + end + local outdata = os.iorun("xmake") + if outdata then + if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then + raise("Modules incremental compilation does not work\n%s", outdata) + end + end +end + +function main(t) + -- TODO c++ modules with pch does not work for gcc now. + if is_host("linux") then + local clang = find_tool("clang", {version = true}) + if clang then + os.exec("xmake f --toolchain=clang -c --yes --policies=build.c++.modules.std:n,build.c++.clang.fallbackscanner") + _build() + end + else + _build() + end +end diff --git a/tests/projects/other/build_deps/xmake.lua b/tests/projects/other/build_deps/xmake.lua index 5729b5c8f..b2324e3dd 100644 --- a/tests/projects/other/build_deps/xmake.lua +++ b/tests/projects/other/build_deps/xmake.lua @@ -4,7 +4,7 @@ target("dep1") set_kind("static") add_deps("dep3") add_files("src/dep1.c") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep3"):targetfile()) @@ -21,7 +21,7 @@ target("dep2") set_kind("static") add_deps("dep3") add_files("src/dep2.c") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep3"):targetfile()) @@ -38,7 +38,7 @@ target("dep3") set_kind("static") add_files("src/dep3.c") add_deps("dep4", "dep5") - set_policy("build.across_targets_in_parallel", false) + set_policy("build.fence", true) after_load(function (target) os.rm(target:targetfile()) os.rm(target:dep("dep4"):targetfile()) diff --git a/tests/projects/other/merge_archive/xmake.lua b/tests/projects/other/merge_archive/xmake.lua index 11092dec9..7cf74471d 100644 --- a/tests/projects/other/merge_archive/xmake.lua +++ b/tests/projects/other/merge_archive/xmake.lua @@ -3,18 +3,19 @@ add_rules("mode.debug", "mode.release") target("add") set_kind("static") add_files("src/add.c") + set_policy("build.fence", true) set_targetdir("$(buildir)/merge_archive") target("sub") set_kind("static") add_files("src/sub.c") + set_policy("build.fence", true) set_targetdir("$(buildir)/merge_archive") target("mul") set_kind("static") add_deps("add", "sub") add_files("src/mul.c") - set_policy("build.across_targets_in_parallel", false) if is_plat("windows") then add_files("$(buildir)/merge_archive/*.lib") else diff --git a/tests/projects/other/merge_object/xmake.lua b/tests/projects/other/merge_object/xmake.lua index b8799a0b8..bbd7c1f98 100644 --- a/tests/projects/other/merge_object/xmake.lua +++ b/tests/projects/other/merge_object/xmake.lua @@ -1,9 +1,9 @@ add_rules("mode.debug", "mode.release") -set_policy("build.across_targets_in_parallel", false) target("merge_object") set_kind("static") add_files("src/interface.c") + set_policy("build.fence", true) after_build_file(function (target, sourcefile) os.cp(target:objectfile(sourcefile), "$(buildir)/merge_object/") end) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 28a33db78..d4f5658fb 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -22,302 +22,41 @@ import("core.base.option") import("core.project.config") import("core.project.project") -import("private.async.jobpool") -import("async.runjobs") -import("private.utils.batchcmds") -import("core.base.hashset") -import("private.service.remote_cache.client", {alias = "remote_cache_client"}) import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_buildutils"}) +import("deprecated.build", {alias = "deprecated_build"}) --- clean target for rebuilding -function _clean_target(target) - if target:targetfile() then - os.tryrm(target:symbolfile()) - os.tryrm(target:targetfile()) - end -end - --- add builtin batch jobs -function _add_batchjobs_builtin(batchjobs, rootjob, target) - - -- uses the rules script? - local job, job_leaf - for _, r in irpairs(target:orderules()) do -- reverse rules order for batchjobs:addjob() - local script = r:script("build") - if script then - if r:extraconf("build", "batch") then - job, job_leaf = assert(script(target, batchjobs, {rootjob = job or rootjob}), "rule(%s):on_build(): no returned job!", r:name()) - else - job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) - script(target, {progress = opt.progress}) - end, {rootjob = job or rootjob}) - end - else - local buildcmd = r:script("buildcmd") - if buildcmd then - job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) - local batchcmds_ = batchcmds.new({target = target}) - buildcmd(target, batchcmds_, {progress = opt.progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = job or rootjob}) - end - end - end - - -- uses the builtin target script - if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly()) then - job, job_leaf = import("kinds." .. target:kind(), {anonymous = true})(batchjobs, rootjob, target) - end - job = job or rootjob - return job, job_leaf or job -end - --- add batch jobs -function _add_batchjobs(batchjobs, rootjob, target) - - local job, job_leaf - local script = target:script("build") - if not script then - -- do builtin batch jobs - job, job_leaf = _add_batchjobs_builtin(batchjobs, rootjob, target) - elseif target:extraconf("build", "batch") then - -- do custom batch script - -- e.g. - -- target("test") - -- on_build(function (target, batchjobs, opt) - -- return batchjobs:addjob("test", function (idx, total) - -- print("build it") - -- end, {rootjob = opt.rootjob}) - -- end, {batch = true}) - -- - job, job_leaf = assert(script(target, batchjobs, {rootjob = rootjob}), "target(%s):on_build(): no returned job!", target:name()) - else - -- do custom script directly - -- e.g. - -- - -- target("test") - -- on_build(function (target, opt) - -- print("build it") - -- end) - -- - job = batchjobs:addjob(target:name() .. "/build", function (index, total, opt) - script(target, {progress = opt.progress}) - end, {rootjob = rootjob}) - end - return job, job_leaf or job +-- run prepare jobs +function _prepare(targets_root, opt) + opt = opt or {} + opt.job_kind = "prepare" + opt.progress_factor = 0.05 + target_buildutils.run_targetjobs(targets_root, opt) end --- add batch jobs for the given target -function _add_batchjobs_for_target(batchjobs, rootjob, target) - - -- has been disabled? - if not target:is_enabled() then - return - end - - -- add after_build job for target - local pkgenvs = _g.pkgenvs or {} - _g.pkgenvs = pkgenvs - local job_build_after = batchjobs:addjob(target:name() .. "/after_build", function (index, total, opt) - - -- do after_build - local progress = opt.progress - local after_build = target:script("build_after") - if after_build then - after_build(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local after_build = r:script("build_after") - if after_build then - after_build(target, {progress = progress}) - else - local after_buildcmd = r:script("buildcmd_after") - if after_buildcmd then - local batchcmds_ = batchcmds.new({target = target}) - after_buildcmd(target, batchcmds_, {progress = progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - end - - -- restore environments - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = nil - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - end, {rootjob = rootjob}) - - -- add batch jobs for target, @note only on_build script support batch jobs - local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_build_after, target) - - -- add before_build job for target - local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total, opt) - - -- enter package environments - -- https://github.com/xmake-io/xmake/issues/4033 - -- - -- maybe mixing envs isn't a great solution, - -- but it's the most efficient compromise compared to setting envs in every on_build_file. - -- - if target:pkgenvs() then - pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() - pkgenvs.newenvs = pkgenvs.newenvs or {} - pkgenvs.newenvs[target] = target:pkgenvs() - local newenvs = pkgenvs.oldenvs - for _, envs in pairs(pkgenvs.newenvs) do - newenvs = os.joinenvs(envs, newenvs) - end - os.setenvs(newenvs) - end - - -- clean target if rebuild - if target:is_rebuilt() and not option.get("dry-run") then - _clean_target(target) - end - - -- do before_build - -- we cannot add batchjobs for this rule scripts, @see https://github.com/xmake-io/xmake/issues/2684 - local progress = opt.progress - local before_build = target:script("build_before") - if before_build then - before_build(target, {progress = progress}) - end - for _, r in ipairs(target:orderules()) do - local before_build = r:script("build_before") - if before_build then - before_build(target, {progress = progress}) - else - local before_buildcmd = r:script("buildcmd_before") - if before_buildcmd then - local batchcmds_ = batchcmds.new({target = target}) - before_buildcmd(target, batchcmds_, {progress = progress}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end - end - end - end, {rootjob = job_build_leaf}) - return job_build_before, job_build, job_build_after -end - --- add batch jobs for the given target and deps -function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, jobrefs_before) - local targetjob_ref = jobrefs[target:name()] - if targetjob_ref then - batchjobs:add(targetjob_ref, rootjob) - else - local job_build_before, job_build, job_build_after = _add_batchjobs_for_target(batchjobs, rootjob, target) - if job_build_before and job_build and job_build_after then - jobrefs[target:name()] = job_build_after - jobrefs_before[target:name()] = job_build_before - for _, depname in ipairs(target:get("deps")) do - local dep = project.target(depname, {namespace = target:namespace()}) - local targetjob = job_build - -- @see https://github.com/xmake-io/xmake/discussions/2500 - if dep:policy("build.across_targets_in_parallel") == false then - targetjob = job_build_before - end - _add_batchjobs_for_target_and_deps(batchjobs, targetjob, dep, jobrefs, jobrefs_before) - end - end +-- run build jobs +function _build(targets_root, opt) + opt = opt or {} + opt.job_kind = "build" + opt.progress_factor = 0.95 + if distcc_build_client.is_connected() then + opt.distcc = distcc_build_client.singleton() end + target_buildutils.run_targetjobs(targets_root, opt) end --- get batch jobs, @note we need to export it for private.diagnosis.dump_buildjobs -function get_batchjobs(targetnames, group_pattern) +function main(targetnames, opt) -- get root targets - local targets_root = {} - if targetnames then - for _, targetname in ipairs(table.wrap(targetnames)) do - local target = project.target(targetname) - if target then - table.insert(targets_root, target) - if option.get("rebuild") then - target:data_set("rebuilt", true) - if not option.get("shallow") then - for _, dep in ipairs(target:orderdeps()) do - dep:data_set("rebuilt", true) - end - end - end - end - end - else - local depset = hashset.new() - local targets = {} - for _, target in ipairs(project.ordertargets()) do - if target:is_enabled() then - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - end - for _, target in ipairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - if option.get("rebuild") then - target:data_set("rebuilt", true) - end - end - end + local targets_root = target_buildutils.get_root_targets(targetnames, opt) - -- generate batch jobs for default or all targets - local jobrefs = {} - local jobrefs_before = {} - local batchjobs = jobpool.new() - for _, target in ipairs(targets_root) do - _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), target, jobrefs, jobrefs_before) - end + -- prepare to build + _prepare(targets_root, opt) - -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 - for _, target in ipairs(project.ordertargets()) do - local target_job_before = jobrefs_before[target:name()] - if target_job_before then - for _, dep in ipairs(target:orderdeps()) do - if dep:policy("build.fence") then - local fence_job = jobrefs[dep:name()] - if fence_job then - batchjobs:add(fence_job, target_job_before) - end - end - end - end - end - - return batchjobs -end - --- the main entry -function main(targetnames, group_pattern) - - -- enable distcc? - local distcc - if distcc_build_client.is_connected() then - distcc = distcc_build_client.singleton() - end - - -- build all jobs - local batchjobs = get_batchjobs(targetnames, group_pattern) - if batchjobs and batchjobs:size() > 0 then - local curdir = os.curdir() - runjobs("build", batchjobs, {on_exit = function (errors) - import("utils.progress") - if errors and progress.showing_without_scroll() then - print("") - end - end, comax = option.get("jobs") or 1, curdir = curdir, distcc = distcc}) - os.cd(curdir) + -- do build + if project.policy("build.jobgraph") then + _build(targets_root, opt) + else + deprecated_build(targets_root, opt) end end diff --git a/xmake/actions/build/build_files.lua b/xmake/actions/build/build_files.lua index ad85d8b2a..1bbad79f6 100644 --- a/xmake/actions/build/build_files.lua +++ b/xmake/actions/build/build_files.lua @@ -23,131 +23,9 @@ import("core.base.option") import("core.base.hashset") import("core.project.config") import("core.project.project") -import("private.async.jobpool") -import("async.runjobs") -import("kinds.object") - --- match source files -function _match_sourcefiles(sourcefile, filepatterns) - for _, filepattern in ipairs(filepatterns) do - if sourcefile:match(filepattern.pattern) == sourcefile then - if filepattern.excludes then - if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then - sourcefile = sourcefile:sub(#filepattern.rootdir + 2) - end - for _, exclude in ipairs(filepattern.excludes) do - if sourcefile:match(exclude) == sourcefile then - return false - end - end - end - return true - end - end -end - --- add batch jobs -function _add_batchjobs(batchjobs, rootjob, target, filepatterns) - - local newbatches = {} - local sourcecount = 0 - for rulename, sourcebatch in pairs(target:sourcebatches()) do - local objectfiles = sourcebatch.objectfiles - local dependfiles = sourcebatch.dependfiles - local sourcekind = sourcebatch.sourcekind - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do - if _match_sourcefiles(sourcefile, filepatterns) then - local newbatch = newbatches[rulename] - if not newbatch then - newbatch = {} - newbatch.sourcekind = sourcekind - newbatch.rulename = rulename - newbatch.sourcefiles = {} - end - table.insert(newbatch.sourcefiles, sourcefile) - if objectfiles then - newbatch.objectfiles = newbatch.objectfiles or {} - table.insert(newbatch.objectfiles, objectfiles[idx]) - end - if dependfiles then - newbatch.dependfiles = newbatch.dependfiles or {} - table.insert(newbatch.dependfiles, dependfiles[idx]) - end - newbatches[rulename] = newbatch - sourcecount = sourcecount + 1 - end - end - end - if sourcecount > 0 then - return object.add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, newbatches) - end -end - --- add batch jobs for the given target -function _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) - - -- has been disabled? - if not target:is_enabled() then - return - end - - -- add batch jobs for target - return _add_batchjobs(batchjobs, rootjob, target, filepatterns) -end - --- add batch jobs for the given target and deps -function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, filepatterns) - local targetjob_ref = jobrefs[target:name()] - if targetjob_ref then - batchjobs:add(targetjob_ref, rootjob) - else - local targetjob, targetjob_root = _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) - if targetjob and targetjob_root then - jobrefs[target:name()] = targetjob_root - if not option.get("shallow") then - for _, depname in ipairs(target:get("deps")) do - _add_batchjobs_for_target_and_deps(batchjobs, targetjob, jobrefs, - project.target(depname, {namespace = target:namespace()}), filepatterns) - end - end - end - end -end - --- get batch jobs -function _get_batchjobs(targetname, group_pattern, filepatterns) - - -- get root targets - local targets_root = {} - if targetname then - table.insert(targets_root, project.target(targetname)) - else - local depset = hashset.new() - local targets = {} - for _, target in pairs(project.targets()) do - local group = target:get("group") - if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then - for _, depname in ipairs(target:get("deps")) do - depset:insert(depname) - end - table.insert(targets, target) - end - end - for _, target in pairs(targets) do - if not depset:has(target:name()) then - table.insert(targets_root, target) - end - end - end - - -- generate batch jobs for default or all targets - local jobrefs = {} - local batchjobs = jobpool.new() - for _, target in pairs(targets_root) do - _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), jobrefs, target, filepatterns) - end - return batchjobs -end +import("private.service.distcc_build.client", {alias = "distcc_build_client"}) +import("private.action.build.target", {alias = "target_buildutils"}) +import("deprecated.build_files", {alias = "deprecated_build_files"}) -- convert all sourcefiles to lua pattern function _get_file_patterns(sourcefiles) @@ -193,20 +71,42 @@ function _get_file_patterns(sourcefiles) return patterns end --- the main entry -function main(targetname, group_pattern, sourcefiles) +-- run prepare files jobs +function _prepare_files(targets_root, opt) + opt = opt or {} + opt.job_kind = "prepare" + opt.progress_factor = 0.05 + opt.filepatterns = _get_file_patterns(opt.sourcefiles) + target_buildutils.run_filejobs(targets_root, opt) +end + +-- run build files jobs +function _build_files(targets_root, opt) + opt = opt or {} + opt.job_kind = "build" + opt.progress_factor = 0.95 + opt.filepatterns = _get_file_patterns(opt.sourcefiles) + if distcc_build_client.is_connected() then + opt.distcc = distcc_build_client.singleton() + end + if not target_buildutils.run_filejobs(targets_root, opt) then + wprint("%s not found!", opt.sourcefiles) + end +end + +function main(targetnames, opt) + + -- get root targets + local targets_root = target_buildutils.get_root_targets(targetnames, opt) - -- convert all sourcefiles to lua pattern - local filepatterns = _get_file_patterns(sourcefiles) + -- prepare to build files + _prepare_files(targets_root, opt) - -- build all jobs - local batchjobs = _get_batchjobs(targetname, group_pattern, filepatterns) - if batchjobs and batchjobs:size() > 0 then - local curdir = os.curdir() - runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) - os.cd(curdir) + -- do build files + if project.policy("build.jobgraph") then + _build_files(targets_root, opt) else - wprint("%s not found!", sourcefiles) + deprecated_build_files(targets_root, opt) end end diff --git a/xmake/actions/build/deprecated/build.lua b/xmake/actions/build/deprecated/build.lua new file mode 100644 index 000000000..377ab5e18 --- /dev/null +++ b/xmake/actions/build/deprecated/build.lua @@ -0,0 +1,282 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build.lua +-- + +-- imports +import("core.base.option") +import("core.project.config") +import("core.project.project") +import("private.async.jobpool") +import("async.runjobs") +import("private.utils.batchcmds") +import("core.base.hashset") +import("private.service.distcc_build.client", {alias = "distcc_build_client"}) + +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + +-- add builtin batch jobs +function _add_batchjobs_builtin(batchjobs, rootjob, target) + + -- uses the rules script? + local job, job_leaf + for _, r in irpairs(target:orderules()) do -- reverse rules order for batchjobs:addjob() + local script = r:script("build") + if script then + if r:extraconf("build", "batch") then + job, job_leaf = assert(script(target, batchjobs, {rootjob = job or rootjob}), "rule(%s):on_build(): no returned job!", r:name()) + elseif r:extraconf("build", "jobgraph") then + wprint("rule(%s) with jobgraph found, please enable `build.jobgraph` policy first!", r:name()) + else + job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) + script(target, {progress = opt.progress}) + end, {rootjob = job or rootjob}) + end + else + local buildcmd = r:script("buildcmd") + if buildcmd then + job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total, opt) + local batchcmds_ = batchcmds.new({target = target}) + buildcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = job or rootjob}) + end + end + end + + -- uses the builtin target script + if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly()) then + job, job_leaf = import("kinds." .. target:kind(), {anonymous = true})(batchjobs, rootjob, target) + end + job = job or rootjob + return job, job_leaf or job +end + +-- add batch jobs +function _add_batchjobs(batchjobs, rootjob, target) + + local job, job_leaf + local script = target:script("build") + if not script then + -- do builtin batch jobs + job, job_leaf = _add_batchjobs_builtin(batchjobs, rootjob, target) + elseif target:extraconf("build", "batch") then + -- do custom batch script + -- e.g. + -- target("test") + -- on_build(function (target, batchjobs, opt) + -- return batchjobs:addjob("test", function (idx, total) + -- print("build it") + -- end, {rootjob = opt.rootjob}) + -- end, {batch = true}) + -- + job, job_leaf = assert(script(target, batchjobs, {rootjob = rootjob}), "target(%s):on_build(): no returned job!", target:name()) + else + -- do custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- print("build it") + -- end) + -- + job = batchjobs:addjob(target:name() .. "/build", function (index, total, opt) + script(target, {progress = opt.progress}) + end, {rootjob = rootjob}) + end + return job, job_leaf or job +end + +-- add batch jobs for the given target +function _add_batchjobs_for_target(batchjobs, rootjob, target) + + -- has been disabled? + if not target:is_enabled() then + return + end + + -- add after_build job for target + local pkgenvs = _g.pkgenvs or {} + _g.pkgenvs = pkgenvs + local job_build_after = batchjobs:addjob(target:name() .. "/after_build", function (index, total, opt) + + -- do after_build + local progress = opt.progress + local after_build = target:script("build_after") + if after_build then + after_build(target, {progress = progress}) + end + for _, r in ipairs(target:orderules()) do + local after_build = r:script("build_after") + if after_build then + after_build(target, {progress = progress}) + else + local after_buildcmd = r:script("buildcmd_after") + if after_buildcmd then + local batchcmds_ = batchcmds.new({target = target}) + after_buildcmd(target, batchcmds_, {progress = progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + end + + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + end, {rootjob = rootjob}) + + -- add batch jobs for target, @note only on_build script support batch jobs + local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_build_after, target) + + -- add before_build job for target + local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total, opt) + + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target if rebuild + if target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + + -- do before_build + -- we cannot add batchjobs for this rule scripts, @see https://github.com/xmake-io/xmake/issues/2684 + local progress = opt.progress + local before_build = target:script("build_before") + if before_build then + before_build(target, {progress = progress}) + end + for _, r in ipairs(target:orderules()) do + local before_build = r:script("build_before") + if before_build then + before_build(target, {progress = progress}) + else + local before_buildcmd = r:script("buildcmd_before") + if before_buildcmd then + local batchcmds_ = batchcmds.new({target = target}) + before_buildcmd(target, batchcmds_, {progress = progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end + end + end, {rootjob = job_build_leaf}) + return job_build_before, job_build, job_build_after +end + +-- add batch jobs for the given target and deps +function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, target, jobrefs, jobrefs_before) + local targetjob_ref = jobrefs[target:name()] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local job_build_before, job_build, job_build_after = _add_batchjobs_for_target(batchjobs, rootjob, target) + if job_build_before and job_build and job_build_after then + jobrefs[target:name()] = job_build_after + jobrefs_before[target:name()] = job_build_before + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + local targetjob = job_build + -- @see https://github.com/xmake-io/xmake/discussions/2500 + if dep:policy("build.across_targets_in_parallel") == false then + targetjob = job_build_before + end + _add_batchjobs_for_target_and_deps(batchjobs, targetjob, dep, jobrefs, jobrefs_before) + end + end + end +end + +-- get batch jobs +function _get_batchjobs(targets_root, opt) + + -- generate batch jobs for default or all targets + local jobrefs = {} + local jobrefs_before = {} + local batchjobs = jobpool.new() + for _, target in ipairs(targets_root) do + _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), target, jobrefs, jobrefs_before) + end + + -- add fence jobs, @see https://github.com/xmake-io/xmake/issues/5003 + for _, target in ipairs(project.ordertargets()) do + local target_job_before = jobrefs_before[target:name()] + if target_job_before then + for _, dep in ipairs(target:orderdeps()) do + if dep:policy("build.fence") then + local fence_job = jobrefs[dep:name()] + if fence_job then + batchjobs:add(fence_job, target_job_before) + end + end + end + end + end + + return batchjobs +end + +function main(targets_root, opt) + + -- enable distcc? + local distcc + if distcc_build_client.is_connected() then + distcc = distcc_build_client.singleton() + end + + -- build all jobs + local batchjobs = _get_batchjobs(targets_root, opt) + if batchjobs and batchjobs:size() > 0 then + local curdir = os.curdir() + runjobs("build", batchjobs, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = distcc}) + os.cd(curdir) + end +end diff --git a/xmake/actions/build/deprecated/build_files.lua b/xmake/actions/build/deprecated/build_files.lua new file mode 100644 index 000000000..db58385cb --- /dev/null +++ b/xmake/actions/build/deprecated/build_files.lua @@ -0,0 +1,188 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_files.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.config") +import("core.project.project") +import("private.async.jobpool") +import("async.runjobs") +import("kinds.object") + +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- add batch jobs +function _add_batchjobs(batchjobs, rootjob, target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return object.add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, newbatches) + end +end + +-- add batch jobs for the given target +function _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) + + -- has been disabled? + if not target:is_enabled() then + return + end + + -- add batch jobs for target + return _add_batchjobs(batchjobs, rootjob, target, filepatterns) +end + +-- add batch jobs for the given target and deps +function _add_batchjobs_for_target_and_deps(batchjobs, rootjob, jobrefs, target, filepatterns) + local targetjob_ref = jobrefs[target:name()] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local targetjob, targetjob_root = _add_batchjobs_for_target(batchjobs, rootjob, target, filepatterns) + if targetjob and targetjob_root then + jobrefs[target:name()] = targetjob_root + if not option.get("shallow") then + for _, depname in ipairs(target:get("deps")) do + _add_batchjobs_for_target_and_deps(batchjobs, targetjob, jobrefs, + project.target(depname, {namespace = target:namespace()}), filepatterns) + end + end + end + end +end + +-- get batch jobs +function _get_batchjobs(targets_root, opt) + + -- convert all sourcefiles to lua pattern + local filepatterns = _get_file_patterns(opt.sourcefiles) + + -- generate batch jobs for default or all targets + local jobrefs = {} + local batchjobs = jobpool.new() + for _, target in pairs(targets_root) do + _add_batchjobs_for_target_and_deps(batchjobs, batchjobs:rootjob(), jobrefs, target, filepatterns) + end + return batchjobs +end + +-- convert all sourcefiles to lua pattern +function _get_file_patterns(sourcefiles) + local patterns = {} + for _, sourcefile in ipairs(path.splitenv(sourcefiles)) do + + -- get the excludes + local pattern = sourcefile:trim() + local excludes = pattern:match("|.*$") + if excludes then excludes = excludes:split("|", {plain = true}) end + + -- translate excludes + if excludes then + local _excludes = {} + for _, exclude in ipairs(excludes) do + exclude = path.translate(exclude) + exclude = path.pattern(exclude) + table.insert(_excludes, exclude) + end + excludes = _excludes + end + + -- translate path and remove some repeat separators + pattern = path.translate(pattern:gsub("|.*$", "")) + + -- remove "./" or '.\\' prefix + if pattern:sub(1, 2):find('%.[/\\]') then + pattern = pattern:sub(3) + end + + -- get the root directory + local rootdir = pattern + local startpos = pattern:find("*", 1, true) + if startpos then + rootdir = rootdir:sub(1, startpos - 1) + end + rootdir = path.directory(rootdir) + + -- convert to lua path pattern + pattern = path.pattern(pattern) + table.insert(patterns, {pattern = pattern, excludes = excludes, rootdir = rootdir}) + end + return patterns +end + +function main(targets_root, opt) + + -- build all jobs + local batchjobs = _get_batchjobs(targets_root, opt) + if batchjobs and batchjobs:size() > 0 then + local curdir = os.curdir() + runjobs("build_files", batchjobs, {comax = option.get("jobs") or 1, curdir = curdir}) + os.cd(curdir) + else + wprint("%s not found!", opt.sourcefiles) + end +end + + diff --git a/xmake/actions/build/kinds/binary.lua b/xmake/actions/build/deprecated/kinds/binary.lua index 3f67232f7..34ef272a9 100644 --- a/xmake/actions/build/kinds/binary.lua +++ b/xmake/actions/build/deprecated/kinds/binary.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/kinds/moduleonly.lua b/xmake/actions/build/deprecated/kinds/moduleonly.lua index d177b8b97..d177b8b97 100644 --- a/xmake/actions/build/kinds/moduleonly.lua +++ b/xmake/actions/build/deprecated/kinds/moduleonly.lua diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/deprecated/kinds/object.lua index b5359f79e..c4ed8721a 100644 --- a/xmake/actions/build/kinds/object.lua +++ b/xmake/actions/build/deprecated/kinds/object.lua @@ -25,7 +25,7 @@ import("core.project.config") import("core.project.project") import("async.runjobs") import("private.utils.batchcmds") -import("private.utils.rule_groups") +import("rule_groups") -- has scripts for the custom rule function _has_scripts_for_rule(ruleinst, suffix) diff --git a/xmake/modules/private/utils/rule_groups.lua b/xmake/actions/build/deprecated/kinds/rule_groups.lua index d64a75d62..d64a75d62 100644 --- a/xmake/modules/private/utils/rule_groups.lua +++ b/xmake/actions/build/deprecated/kinds/rule_groups.lua diff --git a/xmake/actions/build/kinds/shared.lua b/xmake/actions/build/deprecated/kinds/shared.lua index cc7aa010c..921a8e502 100644 --- a/xmake/actions/build/kinds/shared.lua +++ b/xmake/actions/build/deprecated/kinds/shared.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/kinds/static.lua b/xmake/actions/build/deprecated/kinds/static.lua index abb6c5f72..cd7e07ae1 100644 --- a/xmake/actions/build/kinds/static.lua +++ b/xmake/actions/build/deprecated/kinds/static.lua @@ -27,7 +27,7 @@ import("core.project.depend") import("utils.progress") import("private.utils.batchcmds") import("object", {alias = "object_target"}) -import("linkdepfiles", {alias = "get_linkdepfiles"}) +import("private.action.build.target", {alias = "target_buildutils"}) -- do link target function _do_link_target(target, opt) @@ -35,7 +35,7 @@ function _do_link_target(target, opt) local linkflags = linkinst:linkflags({target = target}) -- need build this target? - local depfiles = get_linkdepfiles(target) + local depfiles = target_buildutils.get_linkdepfiles(target) local dryrun = option.get("dry-run") local depvalues = {linkinst:program(), linkflags} depend.on_changed(function () diff --git a/xmake/actions/build/main.lua b/xmake/actions/build/main.lua index fe7fcc771..d9ad6818f 100644 --- a/xmake/actions/build/main.lua +++ b/xmake/actions/build/main.lua @@ -101,20 +101,18 @@ function _do_project_rules(scriptname, opt) end -- do build -function _do_build(targetname, group_pattern) +function _do_build(targetname, opt) local sourcefiles = option.get("files") if sourcefiles then - build_files(targetname, group_pattern, sourcefiles) + build_files(targetname, {group_pattern = opt.group_pattern, sourcefiles = sourcefiles}) else - build(targetname, group_pattern) + build(targetname, {group_pattern = opt.group_pattern}) end end -- build targets function build_targets(targetnames, opt) opt = opt or {} - - local group_pattern = opt.group_pattern try { function () @@ -123,7 +121,7 @@ function build_targets(targetnames, opt) _do_project_rules("build_before") -- do build - _do_build(targetnames, group_pattern) + _do_build(targetnames, opt) -- do check check_targets(targetnames, {build = true}) @@ -146,8 +144,8 @@ function build_targets(targetnames, opt) -- raise if errors then raise(errors) - elseif group_pattern then - raise("build targets with group(%s) failed!", group_pattern) + elseif opt.group_pattern then + raise("build targets with group(%s) failed!", opt.group_pattern) elseif targetnames then targetnames = table.wrap(targetnames) raise("build target: %s failed!", table.concat(targetnames, ", ")) @@ -162,7 +160,8 @@ function build_targets(targetnames, opt) _do_project_rules("build_after") end -function main() +function main(opt) + opt = opt or {} -- try building it using third-party buildsystem if xmake.lua not exists if not os.isfile(project.rootfile()) and _try_build() then @@ -208,9 +207,11 @@ function main() project.unlock() -- trace - local str = "" - if build_time then - str = string.format(", spent %ss", build_time / 1000) + if not opt.disable_dump then + local str = "" + if build_time then + str = string.format(", spent %ss", build_time / 1000) + end + progress.show(100, "${color.success}build ok%s", str) end - progress.show(100, "${color.success}build ok%s", str) end diff --git a/xmake/actions/config/main.lua b/xmake/actions/config/main.lua index 0f69142a5..899748d2e 100644 --- a/xmake/actions/config/main.lua +++ b/xmake/actions/config/main.lua @@ -95,6 +95,7 @@ function _need_check(changed) if not changed then if os.mtime(path.join(os.programdir(), "core", "main.lua")) > os.mtime(config.filepath()) then changed = true + os.touch(config.filepath(), {mtime = os.time()}) end end return changed @@ -358,7 +359,6 @@ force to build in current directory via run `xmake -P .`]], os.projectdir()) localcache.clear("option") localcache.clear("package") localcache.clear("toolchain") - localcache.clear("cxxmodules") localcache.set("config", "recheck", true) localcache.save() diff --git a/xmake/actions/run/main.lua b/xmake/actions/run/main.lua index ab2db5cd1..72fe8bbac 100644 --- a/xmake/actions/run/main.lua +++ b/xmake/actions/run/main.lua @@ -196,7 +196,7 @@ function main() -- we need clear the previous config and reload it -- to avoid trigger recheck configs config.clear() - task.run("build", {target = option.get("target"), all = option.get("all")}) + task.run("build", {target = option.get("target"), all = option.get("all")}, {disable_dump = true}) end -- load targets diff --git a/xmake/core/base/graph.lua b/xmake/core/base/graph.lua index 80d4566ec..fd7675628 100644 --- a/xmake/core/base/graph.lua +++ b/xmake/core/base/graph.lua @@ -19,16 +19,19 @@ -- -- load modules -local table = require("base/table") -local object = require("base/object") +local table = require("base/table") +local queue = require("base/queue") +local object = require("base/object") +local hashset = require("base/hashset") +local utils = require("base/utils") -- define module local graph = graph or object { _init = {"_directed"} } {true} -local edge = edge or object { _init = {"_from", "_to", "_weight"} } +local edge = edge or object { _init = {"_from", "_to"} } -- new edge, from -> to -function edge.new(from, to, weight) - return edge {from, to, weight or 1.0} +function edge.new(from, to) + return edge {from, to} end function edge:from() @@ -47,8 +50,8 @@ function edge:other(v) end end -function edge:weight() - return self._weight +function edge:__tostring() + return string.format("<edge:%s-%s>", self:from(), self:to()) end -- clear graph @@ -56,6 +59,10 @@ function graph:clear() self._vertices = {} self._edges = {} self._adjacent_edges = {} + self._edges_map = {} + + -- clear partial topological sort state + self:partial_topo_sort_reset() end -- is empty? @@ -88,6 +95,17 @@ function graph:has_vertex(v) return table.contains(self:vertices(), v) end +-- add an isolated without edges +function graph:add_vertex(v) + if not self:has_vertex(v) then + table.insert(self._vertices, v) + self._adjacent_edges[v] = {} + end + + -- reset partial topological sort state since graph structure changed + self._partial_topo_dirty = true +end + -- remove the given vertex? function graph:remove_vertex(v) local contains = false @@ -98,45 +116,212 @@ function graph:remove_vertex(v) end end) if contains then + self._edges_map[v] = nil self._adjacent_edges[v] = nil -- remove the adjacent edge with this vertex in the other vertices - if not self:is_directed() then - for _, w in ipairs(self:vertices()) do - local edges = self:adjacent_edges(w) - if edges then - table.remove_if(edges, function (_, e) return e:other(w) == v end) + for _, w in ipairs(self:vertices()) do + local edges = self:adjacent_edges(w) + if edges then + table.remove_if(edges, function (_, e) + if e:other(w) == v then + self._edges_map[w] = nil + return true + end + end) + end + end + + -- reset partial topological sort state since graph structure changed + self._partial_topo_dirty = true + end +end + +-- reset partial topological sort state +function graph:partial_topo_sort_reset() + self._partial_topo_in_progress = false + self._partial_topo_in_degree = nil + self._partial_topo_queue = nil + self._partial_topo_processed = nil + self._partial_topo_finished = 0 + self._partial_topo_has_cycle = nil + self._partial_topo_dirty = false +end + +-- get next node in topological order +-- +-- @param limit the maximum number of nodes to return +-- @return array of nodes with zero in-degree, empty when complete +-- @return has_cycle indicates if a cycle was detected +-- +-- @code +-- dag:partial_topo_sort_reset() +-- +-- local node, has_cycle +-- local order_vertices = {} +-- while true do +-- node, has_cycle = dag:partial_topo_sort_next() +-- if node then +-- table.insert(order_vertices, node) +-- dag:partial_topo_sort_remove(node) +-- else +-- if has_cycle then +-- -- find cycle +-- end +-- break +-- end +-- end +-- @endcode +-- +-- e.g. +-- +-- edges: a (indegree: 0) -> b -> c +-- +-- add_edge(a, b) +-- add_edge(b, c) +-- +-- local node1, has_cycle = g:partial_topo_sort_next() -- return a +-- local node2, has_cycle = g:partial_topo_sort_next() -- return b +-- local node3, has_cycle = g:partial_topo_sort_next() -- return c +-- local node4, has_cycle = g:partial_topo_sort_next() -- return nil (empty, all done) +-- +function graph:partial_topo_sort_next() + + -- recompute all nodes if has dirty nodes + if self._partial_topo_dirty then + self:_partial_topo_sort_recompute_dirty() + end + + -- check if we already detected a cycle + if self._partial_topo_has_cycle then + return nil, true + end + + -- initialize topological sort state if not already in progress + if not self._partial_topo_in_progress then + if not self:_partial_topo_sort_init() then + return nil, false + end + self._partial_topo_in_progress = true + end + + -- get one node with zero in-degree + local node + local partial_topo_queue = self._partial_topo_queue + local partial_topo_processed = self._partial_topo_processed + while not partial_topo_queue:empty() do + local v = partial_topo_queue:pop() + if partial_topo_processed:has(v) then + self:partial_topo_sort_remove(v) + else + node = v + partial_topo_processed:insert(node) + break + end + end + + return node, self._partial_topo_has_cycle +end + +-- remove node and update in-degrees based on the nodes in this node +function graph:partial_topo_sort_remove(node) + if node == nil then + return + end + self._partial_topo_finished = self._partial_topo_finished + 1 + local edges = self:adjacent_edges(node) + if edges then + local partial_topo_in_degree = self._partial_topo_in_degree + local partial_topo_queue = self._partial_topo_queue + for _, e in ipairs(edges) do + if e:from() == node then + local w = e:to() + local in_degree = partial_topo_in_degree[w] - 1 + partial_topo_in_degree[w] = in_degree + if in_degree == 0 then + partial_topo_queue:push(w) end end end end + + if self._partial_topo_queue:empty() and self._partial_topo_processed:size() == self._partial_topo_finished then + self._partial_topo_has_cycle = self._partial_topo_finished ~= #self:vertices() + end end --- topological sort -function graph:topological_sort() - local visited = {} +-- topological sort, use kahn's algorithm +-- +-- e.g. +-- +-- edges: a (indegree: 0) -> b -> c +-- +-- add_edge(a, b) +-- add_edge(b, c) +-- +-- it will return {a, b, c} +function graph:topo_sort() + if not self:is_directed() then + return + end + + -- calculate in-degree for each vertex + local in_degree = {} for _, v in ipairs(self:vertices()) do - visited[v] = false + in_degree[v] = 0 end - local order_vertices = {} - local function dfs(v) - visited[v] = true + + -- count incoming edges for each vertex + for _, v in ipairs(self:vertices()) do local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - local w = e:other(v) - if not visited[w] then - dfs(w) + if e:from() == v then + local w = e:to() + in_degree[w] = (in_degree[w] or 0) + 1 end end end - table.insert(order_vertices, v) end + + -- queue of vertices with no incoming edges (no dependencies) + local queue = queue.new() for _, v in ipairs(self:vertices()) do - if not visited[v] then - dfs(v) + if in_degree[v] == 0 then + queue:push(v) end end - return table.reverse(order_vertices) + + -- process queue + local order_vertices = {} + while not queue:empty() do + -- remove a vertex with no incoming edges + local v = queue:pop() + table.insert(order_vertices, v) + + -- for each outgoing edge, remove it and update in-degrees + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + local d = in_degree[w] - 1 + in_degree[w] = d + if d == 0 then + queue:push(w) + end + end + end + end + end + + -- if we couldn't process all vertices, there must be a cycle + local has_cycle = #order_vertices ~= #self:vertices() + return order_vertices, has_cycle +end + +-- deprecated +function graph:topological_sort() + return self:topo_sort() end -- find cycle @@ -189,8 +374,8 @@ function graph:edges() end -- add edge -function graph:add_edge(from, to, weight) - local e = edge.new(from, to, weight) +function graph:add_edge(from, to) + local e = edge.new(from, to) if not self:has_vertex(from) then table.insert(self._vertices, from) self._adjacent_edges[from] = {} @@ -199,22 +384,36 @@ function graph:add_edge(from, to, weight) table.insert(self._vertices, to) self._adjacent_edges[to] = {} end + local edges_map = self._edges_map + edges_map[from] = edges_map[from] or {} + edges_map[from][to] = true if self:is_directed() then - table.insert(self._adjacent_edges[e:from()], e) + table.insert(self._adjacent_edges[from], e) else - table.insert(self._adjacent_edges[e:from()], e) - table.insert(self._adjacent_edges[e:to()], e) + table.insert(self._adjacent_edges[from], e) + table.insert(self._adjacent_edges[to], e) + edges_map[to] = edges_map[to] or {} + edges_map[to][from] = true end table.insert(self._edges, e) + + -- reset partial topological sort state since graph structure changed + self._partial_topo_dirty = true end -- has the given edge? function graph:has_edge(from, to) local edges = self:adjacent_edges(from) if edges then - for _, e in ipairs(edges) do - if e:to() == to then - return true + local edges_map = self._edges_map + local from_map = edges_map[from] + if from_map and from_map[to] then + return true + else + for _, e in ipairs(edges) do + if e:to() == to then + return true + end end end end @@ -228,7 +427,7 @@ function graph:clone() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:from(), e:to(), e:weight()) + gh:add_edge(e:from(), e:to()) end end end @@ -245,7 +444,7 @@ function graph:reverse() local edges = self:adjacent_edges(v) if edges then for _, e in ipairs(edges) do - gh:add_edge(e:to(), e:from(), e:weight()) + gh:add_edge(e:to(), e:from()) end end end @@ -256,18 +455,69 @@ end function graph:dump() local vertices = self:vertices() local edges = self:edges() - print(string.format("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges)) - print("vertices: ") + utils.cprint("graph: %s, vertices: %d, edges: %d", self:is_directed() and "directed" or "not-directed", #vertices, #edges) + utils.cprint("vertices: ") for _, v in ipairs(vertices) do - print(string.format(" %s", v)) + utils.cprint(" %s", v) end - print("") - print("edges: ") + utils.cprint("") + utils.cprint("edges: ") for _, e in ipairs(edges) do - print(string.format(" %s -> %s", e:from(), e:to())) + utils.cprint(" %s ${color.dump.reference}->${clear} %s", e:from(), e:to()) end end +-- initialize topological sort state if not already in progress +function graph:_partial_topo_sort_init() + if not self:is_directed() then + return false + end + + -- calculate in-degree for each vertex + self._partial_topo_in_degree = {} + for _, v in ipairs(self:vertices()) do + self._partial_topo_in_degree[v] = 0 + end + + -- count incoming edges for each vertex + local partial_topo_in_degree = self._partial_topo_in_degree + for _, v in ipairs(self:vertices()) do + local edges = self:adjacent_edges(v) + if edges then + for _, e in ipairs(edges) do + if e:from() == v then + local w = e:to() + partial_topo_in_degree[w] = (partial_topo_in_degree[w] or 0) + 1 + end + end + end + end + + -- initialize queue with vertices that have no incoming edges + self._partial_topo_queue = queue.new() + local partial_topo_queue = self._partial_topo_queue + for _, v in ipairs(self:vertices()) do + if partial_topo_in_degree[v] == 0 then + partial_topo_queue:push(v) + end + end + + self._partial_topo_processed = self._partial_topo_processed or hashset.new() + return true +end + +-- recompute all dirty nodes +-- +-- TODO we recompute all nodes now, but we should optimize to recompute only dirty nodes +function graph:_partial_topo_sort_recompute_dirty() + self._partial_topo_in_progress = false + self._partial_topo_in_degree = nil + self._partial_topo_queue = nil + self._partial_topo_finished = 0 + self._partial_topo_has_cycle = nil + self._partial_topo_dirty = false +end + -- new graph function graph.new(directed) local gh = graph {directed} @@ -277,4 +527,3 @@ end -- return module: graph return graph - diff --git a/xmake/core/base/option.lua b/xmake/core/base/option.lua index 8448788c5..62551dda3 100644 --- a/xmake/core/base/option.lua +++ b/xmake/core/base/option.lua @@ -51,8 +51,6 @@ function option._translate(menu) -- save menu option._MENU = menu - - -- ok return true end diff --git a/xmake/core/base/queue.lua b/xmake/core/base/queue.lua new file mode 100644 index 000000000..dbdc1d87a --- /dev/null +++ b/xmake/core/base/queue.lua @@ -0,0 +1,125 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- load modules +local object = require("base/object") + +-- define module +local queue = queue or object {_init = {"_first", "_last"}} {1, 0} + +-- clear queue +function queue:clear() + self._first = 1 + self._last = 0 +end + +-- push item to queue +function queue:push(item) + local last = self._last + 1 + self._last = last + self[last] = item +end + +-- pop item from queue +function queue:pop() + local first = self._first + if first > self._last then + return nil + end + + local value = self[first] + self[first] = nil + self._first = first + 1 + return value +end + +-- get queue size +function queue:size() + return self._last - self._first + 1 +end + +-- is queue empty? +function queue:empty() + return self._first > self._last +end + +-- peek the first item of queue +function queue:first() + if self._first > self._last then + return nil + end + return self[self._first] +end + +-- peek the last item of queue +function queue:last() + if self._first > self._last then + return nil + end + return self[self._last] +end + +-- iterator for all items (forward) +-- +-- e.g. +-- +-- for item in queue:items() do +-- print(item) +-- end +-- +function queue:items() + local index = self._first - 1 + local last = self._last + return function() + index = index + 1 + if index <= last then + return self[index] + end + end +end + +-- iterator for all items (reverse) +function queue:ritems() + local index = self._last + 1 + local first = self._first + return function() + index = index - 1 + if index >= first then + return self[index] + end + end +end + +-- clone queue +function queue:clone() + local q = queue.new() + for i = self._first, self._last do + q:push(self[i]) + end + return q +end + +-- new queue +function queue.new() + return queue() +end + +-- return module: queue +return queue diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index c3754d540..c4da9fdd3 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -94,10 +94,12 @@ function policy.policies() ["build.c++.gcc.modules.cxx11abi"] = {description = "Force to enable new cxx11 abi in C++ modules for gcc.", type = "boolean"}, -- Enable cuda device link ["build.cuda.devlink"] = {description = "Enable Cuda devlink.", type = "boolean"}, + -- Enable build jobgraph + ["build.jobgraph"] = {description = "Enable build jobgraph.", default = true, type = "boolean"}, -- Enable windows UAC and set level, e.g. invoker, admin, highest ["windows.manifest.uac"] = {description = "Enable windows manifest UAC.", type = "string"}, -- Enable ui access for windows UAC - ["windows.manifest.uac.ui"] = {description = "Enable windows manifest UAC.", type = "boolean"}, + ["windows.manifest.uac.ui"] = {description = "Enable ui access for windows UAC.", type = "boolean"}, -- Automatically build before running ["run.autobuild"] = {description = "Automatically build before running.", type = "boolean"}, -- Enable install rpath diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua index 7b42d47ea..956e3c41d 100644 --- a/xmake/core/project/project.lua +++ b/xmake/core/project/project.lua @@ -885,6 +885,7 @@ function project._init_default_policies() if semver.compare(compatibility_version, "3.0") >= 0 then policy.set_default("package.cmake_generator.ninja", true) policy.set_default("build.c++.msvc.runtime", "MD") + policy.set_default("run.autobuild", true) else policy.set_default("package.cmake_generator.ninja", false) policy.set_default("build.c++.msvc.runtime", "MT") diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index 571790fc5..89ee45f8b 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -61,6 +61,18 @@ function _instance:_build_deps() self._DEPS = self._DEPS or {} self._ORDERDEPS = self._ORDERDEPS or {} instance_deps.load_deps(self, instances, self._DEPS, self._ORDERDEPS, {self:fullname()}) + + -- compatible with `add_deps("foo", {order = true})` + local plaindeps = self:get("deps") + if plaindeps then + for _, depname in ipairs(table.wrap(plaindeps)) do + if self:extraconf("deps", depname, "order") then + self:add("orders", depname, self:name()) + utils.warning("add_deps(%s, {order = true}) has been deprecated, please use `add_orders(%s, %s) instead of it`", + depname, depname, self:name()) + end + end + end end -- clone rule @@ -280,6 +292,11 @@ function rule.apis() , "rule.add_deps" , "rule.add_imports" } + , groups = + { + -- rule.add_xxx + "rule.add_orders" + } , script = { -- rule.on_xxx @@ -287,6 +304,9 @@ function rule.apis() , "rule.on_test" , "rule.on_load" , "rule.on_config" + , "rule.on_prepare" + , "rule.on_prepare_file" + , "rule.on_prepare_files" , "rule.on_link" , "rule.on_build" , "rule.on_build_file" @@ -294,18 +314,24 @@ function rule.apis() , "rule.on_clean" , "rule.on_package" , "rule.on_install" - , "rule.on_installcmd" , "rule.on_uninstall" - , "rule.on_uninstallcmd" + , "rule.on_preparecmd" + , "rule.on_preparecmd_file" + , "rule.on_preparecmd_files" , "rule.on_linkcmd" , "rule.on_buildcmd" , "rule.on_buildcmd_file" , "rule.on_buildcmd_files" + , "rule.on_installcmd" + , "rule.on_uninstallcmd" -- rule.before_xxx , "rule.before_run" , "rule.before_test" , "rule.before_load" , "rule.before_config" + , "rule.before_prepare" + , "rule.before_prepare_file" + , "rule.before_prepare_files" , "rule.before_link" , "rule.before_build" , "rule.before_build_file" @@ -313,18 +339,24 @@ function rule.apis() , "rule.before_clean" , "rule.before_package" , "rule.before_install" - , "rule.before_installcmd" , "rule.before_uninstall" - , "rule.before_uninstallcmd" + , "rule.before_preparecmd" + , "rule.before_preparecmd_file" + , "rule.before_preparecmd_files" , "rule.before_linkcmd" , "rule.before_buildcmd" , "rule.before_buildcmd_file" , "rule.before_buildcmd_files" + , "rule.before_installcmd" + , "rule.before_uninstallcmd" -- rule.after_xxx , "rule.after_run" , "rule.after_test" , "rule.after_load" , "rule.after_config" + , "rule.after_prepare" + , "rule.after_prepare_file" + , "rule.after_prepare_files" , "rule.after_link" , "rule.after_build" , "rule.after_build_file" @@ -332,13 +364,16 @@ function rule.apis() , "rule.after_clean" , "rule.after_package" , "rule.after_install" - , "rule.after_installcmd" , "rule.after_uninstall" - , "rule.after_uninstallcmd" + , "rule.after_preparecmd" + , "rule.after_preparecmd_file" + , "rule.after_preparecmd_files" , "rule.after_linkcmd" , "rule.after_buildcmd" , "rule.after_buildcmd_file" , "rule.after_buildcmd_files" + , "rule.after_installcmd" + , "rule.after_uninstallcmd" } } end diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 360edb9cc..089fd1e3c 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -2923,6 +2923,9 @@ function target.apis() , "target.on_test" , "target.on_load" , "target.on_config" + , "target.on_prepare" + , "target.on_prepare_file" + , "target.on_prepare_files" , "target.on_link" , "target.on_build" , "target.on_build_file" @@ -2930,12 +2933,23 @@ function target.apis() , "target.on_clean" , "target.on_package" , "target.on_install" - , "target.on_installcmd" , "target.on_uninstall" + , "target.on_preparecmd" + , "target.on_preparecmd_file" + , "target.on_preparecmd_files" + , "target.on_linkcmd" + , "target.on_buildcmd" + , "target.on_buildcmd_file" + , "target.on_buildcmd_files" + , "target.on_installcmd" , "target.on_uninstallcmd" -- target.before_xxx , "target.before_run" , "target.before_test" + , "target.before_config" + , "target.before_prepare" + , "target.before_prepare_file" + , "target.before_prepare_files" , "target.before_link" , "target.before_build" , "target.before_build_file" @@ -2943,13 +2957,24 @@ function target.apis() , "target.before_clean" , "target.before_package" , "target.before_install" - , "target.before_installcmd" , "target.before_uninstall" + , "target.before_preparecmd" + , "target.before_preparecmd_file" + , "target.before_preparecmd_files" + , "target.before_linkcmd" + , "target.before_buildcmd" + , "target.before_buildcmd_file" + , "target.before_buildcmd_files" + , "target.before_installcmd" , "target.before_uninstallcmd" -- target.after_xxx , "target.after_run" , "target.after_test" , "target.after_load" + , "target.after_config" + , "target.after_prepare" + , "target.after_prepare_file" + , "target.after_prepare_files" , "target.after_link" , "target.after_build" , "target.after_build_file" @@ -2957,8 +2982,15 @@ function target.apis() , "target.after_clean" , "target.after_package" , "target.after_install" - , "target.after_installcmd" , "target.after_uninstall" + , "target.after_preparecmd" + , "target.after_preparecmd_file" + , "target.after_preparecmd_files" + , "target.after_linkcmd" + , "target.after_buildcmd" + , "target.after_buildcmd_file" + , "target.after_buildcmd_files" + , "target.after_installcmd" , "target.after_uninstallcmd" } } diff --git a/xmake/core/sandbox/modules/import/core/base/option.lua b/xmake/core/sandbox/modules/import/core/base/option.lua index 42ac2477a..852115667 100644 --- a/xmake/core/sandbox/modules/import/core/base/option.lua +++ b/xmake/core/sandbox/modules/import/core/base/option.lua @@ -18,33 +18,23 @@ -- @file option.lua -- --- define module -local sandbox_core_base_option = sandbox_core_base_option or {} - -- load modules local table = require("base/table") local option = require("base/option") local raise = require("sandbox/modules/raise") --- get the option value -function sandbox_core_base_option.get(name) - return option.get(name) -end - --- set the option value -function sandbox_core_base_option.set(name, value) - option.set(name, value) -end +-- define module +local sandbox_core_base_option = sandbox_core_base_option or {} --- get the default option value -function sandbox_core_base_option.default(name) - return option.default(name) -end - --- get the given task menu -function sandbox_core_base_option.taskmenu(taskname) - return option.taskmenu(taskname) -end +-- inherit some builtin interfaces +sandbox_core_base_option.get = option.get +sandbox_core_base_option.set = option.set +sandbox_core_base_option.default = option.default +sandbox_core_base_option.save = option.save +sandbox_core_base_option.restore = option.restore +sandbox_core_base_option.boolean = option.boolean +sandbox_core_base_option.taskname = option.taskname +sandbox_core_base_option.taskmenu = option.taskmenu -- get the options function sandbox_core_base_option.options() @@ -68,8 +58,6 @@ end -- parse arguments with the given options function sandbox_core_base_option.raw_parse(argv, options, opt) - - -- check assert(argv and options) -- parse it @@ -82,8 +70,6 @@ end -- parse arguments with the given options function sandbox_core_base_option.parse(argv, options, ...) - - -- check assert(argv and options) -- add common options @@ -117,20 +103,5 @@ function sandbox_core_base_option.parse(argv, options, ...) return results end --- save context -function sandbox_core_base_option.save(taskname) - return option.save(taskname) -end - --- restore context -function sandbox_core_base_option.restore() - option.restore() -end - --- get the boolean value -function sandbox_core_base_option.boolean(value) - return option.boolean(value) -end - -- return module return sandbox_core_base_option diff --git a/xmake/core/sandbox/modules/import/core/base/queue.lua b/xmake/core/sandbox/modules/import/core/base/queue.lua new file mode 100644 index 000000000..aab0a8214 --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/base/queue.lua @@ -0,0 +1,22 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file queue.lua +-- + +-- return module +return require("base/queue") diff --git a/xmake/core/sandbox/modules/import/core/project/project.lua b/xmake/core/sandbox/modules/import/core/project/project.lua index 61b538bb1..c696b79b7 100644 --- a/xmake/core/sandbox/modules/import/core/project/project.lua +++ b/xmake/core/sandbox/modules/import/core/project/project.lua @@ -128,6 +128,10 @@ end -- config target function sandbox_core_project._config_target(target, opt) + local before_config = target:script("config_before") + if before_config then + before_config(target, opt) + end for _, rule in ipairs(table.wrap(target:orderules())) do local before_config = rule:script("config_before") if before_config then @@ -152,9 +156,13 @@ function sandbox_core_project._config_target(target, opt) after_config(target, opt) end end + local config_after = target:script("config_after") + if config_after then + config_after(target, opt) + end end --- config targets +-- config targets, TODO: We should support parallel configuration -- -- @param opt the extra option, e.g. {recheck = false} -- diff --git a/xmake/core/tool/builder.lua b/xmake/core/tool/builder.lua index 2bf4efc1f..0e85911eb 100644 --- a/xmake/core/tool/builder.lua +++ b/xmake/core/tool/builder.lua @@ -668,11 +668,14 @@ function builder:_sort_links_of_items(items, opt) gh:add_edge(k, v) end if not gh:empty() then - local cycle = gh:find_cycle() - if cycle then - utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + local has_cycle + links, has_cycle = gh:topo_sort() + if has_cycle then + local cycle = gh:find_cycle() + if cycle then + utils.warning("cycle links found in add_linkorders(): %s", table.concat(cycle, " -> ")) + end end - links = gh:topological_sort() end end diff --git a/xmake/modules/async/jobgraph.lua b/xmake/modules/async/jobgraph.lua new file mode 100644 index 000000000..1ed5873b7 --- /dev/null +++ b/xmake/modules/async/jobgraph.lua @@ -0,0 +1,263 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file jobgraph.lua +-- + +-- imports +import("core.base.object") +import("core.base.list") +import("core.base.graph") +import("core.base.hashset") + +-- define module +local jobqueue = jobqueue or object {_init = {"_jobgraph", "_dag"}} +local jobgraph = jobgraph or object {_init = {"_name", "_jobs", "_size", "_dag", "_groups"}} + +-- remove the finished job +function jobqueue:remove(job) + local dag = self._dag + dag:partial_topo_sort_remove(job) +end + +-- get a free job from the job queue +function jobqueue:getfree() + local dag = self._dag +::continue:: + local freejob, has_cycle = dag:partial_topo_sort_next() + if has_cycle then + local names = {} + local cycle = dag:find_cycle() + if cycle then + for _, job in ipairs(cycle) do + table.insert(names, job.name) + end + table.insert(names, names[1]) + end + raise("%s: circular job dependency detected!\n%s", self._jobgraph, table.concat(names, "\n -> ")) + end + -- if it's a fake job, we need to skip it and continue to get the next job + if freejob and not freejob.run then + dag:partial_topo_sort_remove(freejob) + goto continue + end + return freejob +end + +-- add a job to the jobgraph +-- +-- e.g. +-- jobgraph:add("xxx", function (index, total, opt) +-- end) +-- +-- @param name the job name +-- @param run the job run command/script +-- @param opt the job options, e.g. {groups = {"xxx"}} +-- +function jobgraph:add(name, run, opt) + opt = opt or {} + local dag = self._dag + local jobs = self._jobs + if not jobs[name] then + local job = {name = name, run = run, distcc = opt.distcc} + jobs[name] = job + dag:add_vertex(job) + self._size = self._size + 1 + + if self._current_groups or opt.groups then + local job_groups = table.join(self._current_groups or {}, opt.groups) + for _, group_name in ipairs(job_groups) do + local groups = self._groups[group_name] + if not groups then + groups = {} + self._groups[group_name] = groups + end + table.insert(groups, job) + end + end + else + raise("job(%s): has already been added!", name) + end +end + +-- remove a given job +function jobgraph:remove(name) + local dag = self._dag + local jobs = self._jobs + local job = jobs[name] + if job then + assert(self._size > 0) + jobs[name] = nil + dag:remove_vertex(job) + self._size = self._size - 1 + end +end + +-- has the given job or group? +function jobgraph:has(name) + return (self._jobs[name] or self._groups[name]) ~= nil +end + +-- enter group to add jobs +-- +-- e.g. +-- jobgraph:group("foo", function () +-- jobgraph:add("job1", function (index, total, opt) +-- TODO +-- end) +-- jobgraph:add("job2", function (index, total, opt) +-- TODO +-- end) +-- end) +function jobgraph:group(name, callback) + local current_groups = self._current_groups + if current_groups == nil then + current_groups = {} + self._current_groups = current_groups + end + table.insert(current_groups, name) + callback() + table.remove(current_groups) +end + +-- add job orders, e.g. add_orders(a, b, c, ...): a -> b -> c, ... +-- +-- and it supports nil, e.g add_orders("foo", nil, "bar", ...) +-- and it also supports to add orders list, e.g. add_orders(orders) +-- +function jobgraph:add_orders(...) + local prev + local prev_is_group + local prev_name + local dag = self._dag + local jobs = self._jobs + local groups = self._groups + local orders = table.pack(...) + local count = orders.n + if count == 1 and type(orders[1]) == "table" then + orders = orders[1] + count = #orders + end + for i = 1, count do + local name = orders[i] + if name then + local curr_is_group = false + local curr = jobs[name] + if not curr then + curr = groups[name] + curr_is_group = true + end + assert(curr, "job(%s) not found in jobgraph(%s)", name, self) + if prev then + if prev_is_group and curr_is_group then + -- we use a bridge job as a node to bridge the two groups. + local bridge = {from_group = prev_name, to_group = name} + for _, job in ipairs(prev) do + if not dag:has_edge(job, bridge) then + dag:add_edge(job, bridge) + end + end + for _, job in ipairs(curr) do + if not dag:has_edge(bridge, job) then + dag:add_edge(bridge, job) + end + end + elseif curr_is_group then + for _, job in ipairs(curr) do + if not dag:has_edge(prev, job) then + dag:add_edge(prev, job) + end + end + elseif prev_is_group then + for _, job in ipairs(prev) do + if not dag:has_edge(job, curr) then + dag:add_edge(job, curr) + end + end + else + if not dag:has_edge(prev, curr) then + dag:add_edge(prev, curr) + end + end + end + prev = curr + prev_is_group = curr_is_group + prev_name = name + end + end +end + +-- build a job queue +function jobgraph:build() + local dag = self._dag + dag:partial_topo_sort_reset() + return jobqueue {self, dag} +end + +-- get jobs +function jobgraph:jobs() + return self._jobs +end + +-- get jobgraph name +function jobgraph:name() + return self._name +end + +-- get job size +function jobgraph:size() + return self._size +end + +-- is empty? +function jobgraph:empty() + return self:size() == 0 +end + +-- dump jobgraph +function jobgraph:dump() + print("================================ %s ================================", self) + for _, node in ipairs(self._dag:vertices()) do + debug.setmetatable(node, {__tostring = function (v) + if v.from_group and v.to_group then + return string.format("${dim}bridge<%s, %s>${clear}", v.from_group, v.to_group) + end + return string.format("${color.dump.string_quote}%s${clear}", v.name) + end}) + end + self._dag:dump() + + print("") + print("groups:") + for name, jobs in pairs(self._groups) do + print(" group(%s):", name) + for _, job in ipairs(jobs) do + cprint(" %s", job) + end + end + print("") +end + +-- tostring +function jobgraph:__tostring() + return string.format("<jobgraph:%s/%d>", self:name() or "anonymous", self:size()) +end + +-- new a jobgraph +function new(name) + return jobgraph {name, {}, 0, graph.new(true), {}} +end diff --git a/xmake/modules/async/runjobs.lua b/xmake/modules/async/runjobs.lua index 0c2a6f366..adf68018d 100644 --- a/xmake/modules/async/runjobs.lua +++ b/xmake/modules/async/runjobs.lua @@ -64,6 +64,11 @@ function main(name, jobs, opt) local group_name = name local jobs_cb = type(jobs) == "function" and jobs or nil assert(timeout < 60000, "runjobs: invalid timeout!") + + -- build jobs queue + if type(jobs) == "table" and jobs.build then + jobs = jobs:build() + end assert(jobs, "runjobs: no jobs!") -- show waiting tips? @@ -158,6 +163,7 @@ function main(name, jobs, opt) local abort_errors local progress_wrapper = {} local job_pending + local progress_factor = opt.progress_factor or 1.0 progress_wrapper.current = function () return count end @@ -166,7 +172,7 @@ function main(name, jobs, opt) end progress_wrapper.percent = function () if total and total > 0 then - return math.floor((count * 100) / total) + return math.floor((count * progress_factor * 100) / total) else return 0 end diff --git a/xmake/modules/cli/amalgamate.lua b/xmake/modules/cli/amalgamate.lua index 8c7ec6fd7..d0a5ba40c 100644 --- a/xmake/modules/cli/amalgamate.lua +++ b/xmake/modules/cli/amalgamate.lua @@ -96,7 +96,7 @@ function _generate_file(target, inputpaths, outputpath, uniqueid) _generate_include_graph(target, inputpaths, gh, {}) -- sort file paths and remove root path - local filepaths = gh:topological_sort() + local filepaths = gh:topo_sort() table.remove(filepaths, 1) -- generate amalgamate file diff --git a/xmake/modules/detect/sdks/find_emsdk.lua b/xmake/modules/detect/sdks/find_emsdk.lua index cbc6998b2..ec8553692 100644 --- a/xmake/modules/detect/sdks/find_emsdk.lua +++ b/xmake/modules/detect/sdks/find_emsdk.lua @@ -33,10 +33,17 @@ function _find_emsdkdir(sdkdir) table.insert(paths, sdkdir) end table.insert(paths, "$(env EMSDK)") + if is_host("linux") then + table.join2(paths, {"/usr/share/emscripten/", "/usr/lib/emscripten/"}) + end local emsdk = find_file("emsdk.py", paths, {suffixes = subdirs}) if emsdk then return path.directory(emsdk) end + local emcc_py = find_file("emcc.py", paths, {suffixes = subdirs}) + if emcc_py then + return path.directory(emcc_py) + end end -- find emsdk @@ -53,6 +60,7 @@ function _find_emsdk(sdkdir) local subdirs = {} table.insert(subdirs, path.join("*", "emscripten")) local emcc = find_file("emcc", sdkdir, {suffixes = subdirs}) + emcc = emcc or find_file("emcc.py", sdkdir) if emcc then emscripten = path.directory(emcc) end diff --git a/xmake/modules/package/manager/cmake/find_package.lua b/xmake/modules/package/manager/cmake/find_package.lua index 6c3a9f194..4c71291d7 100644 --- a/xmake/modules/package/manager/cmake/find_package.lua +++ b/xmake/modules/package/manager/cmake/find_package.lua @@ -83,6 +83,14 @@ function _find_package(cmake, name, opt) cmakefile:print("list(APPEND CMAKE_MODULE_PATH \"%s\")", (moduledir:gsub("\\", "/"))) end end + -- https://github.com/xmake-io/xmake/issues/6296 + local prefixdirs = configs.prefixdirs or opt.prefixdirs + if prefixdirs then + for _, prefixdir in ipairs(prefixdirs) do + cmakefile:print("list(APPEND CMAKE_PREFIX_PATH \"%s\")", (prefixdir:gsub("\\", "/"))) + end + end + -- e.g. set(Boost_USE_STATIC_LIB ON) local presets = configs.presets or opt.presets if presets then diff --git a/xmake/modules/package/tools/autoconf.lua b/xmake/modules/package/tools/autoconf.lua index 8d312a76a..821ab759f 100644 --- a/xmake/modules/package/tools/autoconf.lua +++ b/xmake/modules/package/tools/autoconf.lua @@ -434,6 +434,7 @@ function buildenvs(package, opt) if is_host("windows") then envs.CC = _translate_windows_bin_path(envs.CC) + envs.CXX = _translate_windows_bin_path(envs.CXX) envs.AS = _translate_windows_bin_path(envs.AS) envs.AR = _translate_windows_bin_path(envs.AR) envs.LD = _translate_windows_bin_path(envs.LD) diff --git a/xmake/modules/private/action/build/build_binary.lua b/xmake/modules/private/action/build/build_binary.lua new file mode 100644 index 000000000..3863b8550 --- /dev/null +++ b/xmake/modules/private/action/build/build_binary.lua @@ -0,0 +1,38 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_binary.lua +-- + +-- imports +import("build_object") +import("private.action.build.target", {alias = "target_buildutils"}) + +function main(jobgraph, target, opt) + local objects_group = target:fullname() .. "/objects" + local jobsize = jobgraph:size() + jobgraph:group(objects_group, function () + build_object(jobgraph, target, opt) + end) + if jobgraph:size() > jobsize then + local link_group = target:fullname() .. "/link" + jobgraph:group(link_group, function () + target_buildutils.add_linkjobs(jobgraph, target, opt) + end) + jobgraph:add_orders(objects_group, link_group) + end +end diff --git a/xmake/modules/private/diagnosis/dump_buildjobs.lua b/xmake/modules/private/action/build/build_moduleonly.lua index 16083a40b..32c07375e 100644 --- a/xmake/modules/private/diagnosis/dump_buildjobs.lua +++ b/xmake/modules/private/action/build/build_moduleonly.lua @@ -15,16 +15,12 @@ -- Copyright (C) 2015-present, TBOOX Open Source Group. -- -- @author ruki --- @file dump_buildjobs.lua +-- @file build_moduleonly.lua -- -- imports -import("core.project.config") -import("actions.build.build", {rootdir = os.programdir()}) +import("build_object") --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - print(build.get_batchjobs(targetname)) +function main(jobgraph, target, opt) + build_object(jobgraph, target, opt) end - diff --git a/xmake/actions/build/kinds/linkdepfiles.lua b/xmake/modules/private/action/build/build_object.lua index f96e532fc..08f82092b 100644 --- a/xmake/actions/build/kinds/linkdepfiles.lua +++ b/xmake/modules/private/action/build/build_object.lua @@ -15,25 +15,12 @@ -- Copyright (C) 2015-present, TBOOX Open Source Group. -- -- @author ruki --- @file linkdepfiles.lua +-- @file build_object.lua -- --- get link depfiles -function main(target) - local extrafiles = {} - for _, dep in ipairs(target:orderdeps()) do - if dep:kind() == "static" then - table.insert(extrafiles, dep:targetfile()) - end - end - local linkdepfiles = target:data("linkdepfiles") - if linkdepfiles then - table.join2(extrafiles, linkdepfiles) - end - local objectfiles = target:objectfiles() - local depfiles = objectfiles - if #extrafiles > 0 then - depfiles = table.join(objectfiles, extrafiles) - end - return depfiles +-- imports +import("private.action.build.target", {alias = "target_buildutils"}) + +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) end diff --git a/xmake/modules/private/action/build/build_shared.lua b/xmake/modules/private/action/build/build_shared.lua new file mode 100644 index 000000000..9223093b4 --- /dev/null +++ b/xmake/modules/private/action/build/build_shared.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_shared.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) +end diff --git a/xmake/modules/private/action/build/build_static.lua b/xmake/modules/private/action/build/build_static.lua new file mode 100644 index 000000000..6d47394ed --- /dev/null +++ b/xmake/modules/private/action/build/build_static.lua @@ -0,0 +1,26 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file build_static.lua +-- + +-- imports +import("build_binary") + +function main(jobgraph, target, opt) + build_binary(jobgraph, target, opt) +end diff --git a/xmake/modules/private/action/build/link_objects.lua b/xmake/modules/private/action/build/link_objects.lua new file mode 100644 index 000000000..396896d00 --- /dev/null +++ b/xmake/modules/private/action/build/link_objects.lua @@ -0,0 +1,72 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file link_objects.lua +-- + +-- imports +import("core.base.option") +import("core.tool.linker") +import("core.tool.compiler") +import("core.project.depend") +import("utils.progress") +import("build_object") +import("private.action.build.target", {alias = "target_buildutils"}) + +-- do link target +function _do_link_target(target, opt) + local linkinst = linker.load(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linkinst:linkflags({target = target}) + + -- need build this target? + local depfiles = target_buildutils.get_linkdepfiles(target) + local dryrun = option.get("dry-run") + local depvalues = {linkinst:program(), linkflags} + depend.on_changed(function () + local filename = target:filename() + if target:namespace() then + filename = target:namespace() .. "::" .. filename + end + progress.show(opt.progress, "${color.build.target}linking.$(mode) %s", filename) + + local targetfile = target:targetfile() + local objectfiles = target:objectfiles() + local verbose = option.get("verbose") + if verbose then + -- show the full link command with raw arguments, it will expand @xxx.args for msvc/link on windows + print(linkinst:linkcmd(objectfiles, targetfile, {linkflags = linkflags, rawargs = true})) + end + + if not dryrun then + assert(linkinst:link(objectfiles, targetfile, {linkflags = linkflags})) + end + end, {dependfile = target:dependfile(), + lastmtime = os.mtime(target:targetfile()), + changed = target:is_rebuilt() or option.get("linkonly"), + values = depvalues, files = depfiles, dryrun = dryrun}) +end + +function main(jobgraph, target, opt) + opt = opt or {} + local buildcmds = opt.buildcmds + local linkjob = target:fullname() .. "/link_objects" + jobgraph:add(linkjob, function (index, total, opt) + if not buildcmds then + _do_link_target(target, opt) + end + end) +end diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index ae6bf9f4a..a8b66aa54 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -143,8 +143,8 @@ function build(target, sourcebatch, opt) end end --- add batch jobs to build the source files -function main(target, batchjobs, sourcebatch, opt) +-- add build jobs to batchjobs +function _add_batchjobs(target, batchjobs, sourcebatch, opt) local rootjob = opt.rootjob for i = 1, #sourcebatch.sourcefiles do local sourcefile = sourcebatch.sourcefiles[i] @@ -157,3 +157,27 @@ function main(target, batchjobs, sourcebatch, opt) end, {rootjob = rootjob, distcc = opt.distcc}) end end + +-- add build jobs to jobgraph +function _add_jobgraph(target, jobgraph, sourcebatch, opt) + for i = 1, #sourcebatch.sourcefiles do + local sourcefile = sourcebatch.sourcefiles[i] + local objectfile = sourcebatch.objectfiles[i] + local dependfile = sourcebatch.dependfiles[i] + local sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) + local jobname = target:fullname() .. "/obj/" .. sourcefile + jobgraph:add(jobname, function (index, total, jobopt) + local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = jobopt.progress}, opt) + build_object(target, sourcefile, build_opt) + end, {distcc = opt.distcc}) + end +end + +function main(target, jobgraph, sourcebatch, opt) + opt = opt or {} + if jobgraph.add_orders then + _add_jobgraph(target, jobgraph, sourcebatch, opt) + else + _add_batchjobs(target, jobgraph, sourcebatch, opt) + end +end diff --git a/xmake/modules/private/action/build/pcheader.lua b/xmake/modules/private/action/build/pcheader.lua index 20e857d7f..ec06c8fb8 100644 --- a/xmake/modules/private/action/build/pcheader.lua +++ b/xmake/modules/private/action/build/pcheader.lua @@ -20,7 +20,7 @@ -- imports import("core.language.language") -import("object") +import("object", {alias = "build_objects"}) function config(target, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) @@ -55,7 +55,7 @@ function config(target, langkind, opt) end -- add batch jobs to build the precompiled header file -function build(target, langkind, opt) +function build(target, jobgraph, langkind, opt) local pcheaderfile = target:pcheaderfile(langkind) if pcheaderfile then local sourcefile = pcheaderfile @@ -63,6 +63,6 @@ function build(target, langkind, opt) local dependfile = target:dependfile(objectfile) local sourcekind = language.langkinds()[langkind] local sourcebatch = {sourcekind = sourcekind, sourcefiles = {sourcefile}, objectfiles = {objectfile}, dependfiles = {dependfile}} - object.build(target, sourcebatch, opt) + build_objects(target, jobgraph, sourcebatch, opt) end end diff --git a/xmake/modules/private/action/build/prepare_files.lua b/xmake/modules/private/action/build/prepare_files.lua new file mode 100644 index 000000000..9f0f47ef0 --- /dev/null +++ b/xmake/modules/private/action/build/prepare_files.lua @@ -0,0 +1,27 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file prepare_files.lua +-- + +-- imports +import("core.base.option") +import("private.action.build.target", {alias = "target_buildutils"}) + +function main(jobgraph, target, opt) + target_buildutils.add_filejobs(jobgraph, target, opt) +end diff --git a/xmake/modules/private/action/build/target.lua b/xmake/modules/private/action/build/target.lua new file mode 100644 index 000000000..c302408ac --- /dev/null +++ b/xmake/modules/private/action/build/target.lua @@ -0,0 +1,786 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file target.lua +-- + +-- imports +import("core.base.option") +import("core.base.hashset") +import("core.project.rule") +import("core.project.config") +import("core.project.project") +import("async.runjobs", {alias = "async_runjobs"}) +import("async.jobgraph", {alias = "async_jobgraph"}) +import("private.utils.batchcmds") +import("private.utils.rule", {alias = "rule_utils"}) + +-- clean target for rebuilding +function _clean_target(target) + if target:targetfile() then + os.tryrm(target:symbolfile()) + os.tryrm(target:targetfile()) + end +end + +-- match source files +function _match_sourcefiles(sourcefile, filepatterns) + for _, filepattern in ipairs(filepatterns) do + if sourcefile:match(filepattern.pattern) == sourcefile then + if filepattern.excludes then + if filepattern.rootdir and sourcefile:startswith(filepattern.rootdir) then + sourcefile = sourcefile:sub(#filepattern.rootdir + 2) + end + for _, exclude in ipairs(filepattern.excludes) do + if sourcefile:match(exclude) == sourcefile then + return false + end + end + end + return true + end + end +end + +-- match sourcebatches +function _match_sourcebatches(target, filepatterns) + local newbatches = {} + local sourcecount = 0 + for rulename, sourcebatch in pairs(target:sourcebatches()) do + local objectfiles = sourcebatch.objectfiles + local dependfiles = sourcebatch.dependfiles + local sourcekind = sourcebatch.sourcekind + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + if _match_sourcefiles(sourcefile, filepatterns) then + local newbatch = newbatches[rulename] + if not newbatch then + newbatch = {} + newbatch.sourcekind = sourcekind + newbatch.rulename = rulename + newbatch.sourcefiles = {} + end + table.insert(newbatch.sourcefiles, sourcefile) + if objectfiles then + newbatch.objectfiles = newbatch.objectfiles or {} + table.insert(newbatch.objectfiles, objectfiles[idx]) + end + if dependfiles then + newbatch.dependfiles = newbatch.dependfiles or {} + table.insert(newbatch.dependfiles, dependfiles[idx]) + end + newbatches[rulename] = newbatch + sourcecount = sourcecount + 1 + end + end + end + if sourcecount > 0 then + return newbatches + end +end + +-- add targetjobs and deps orders +function _add_targetjobs_orders(jobgraph, target, dep, opt) + local jobname, jobname_dep + local job_kind = opt.job_kind + if dep:policy("build.fence") or dep:policy("build.across_targets_in_parallel") == false then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + -- build.across_targets_in_parallel is deprecated + if dep:policy("build.across_targets_in_parallel") == false then + wprint("policy(\"build.across_targets_in_parallel\") has been deprecated, please use policy(\"build.fence\") instead of it.") + end + elseif job_kind == "build" then + jobname = target:fullname() .. "/link" + jobname_dep = dep:fullname() .. "/link" + if not jobgraph:has(jobname) then + jobname = string.format("%s/begin_%s", target:fullname(), job_kind) + end + if not jobgraph:has(jobname_dep) then + jobname_dep = string.format("%s/end_%s", dep:fullname(), job_kind) + end + end + if jobname and jobname_dep and jobgraph:has(jobname) and jobgraph:has(jobname_dep) then + jobgraph:add_orders(jobname_dep, jobname) + end +end + +-- add target jobs for the builtin script +function add_targetjobs_for_builtin_script(jobgraph, target, opt) + opt = opt or {} + local job_kind = opt.job_kind + if target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly() then + if job_kind == "prepare" then + import("private.action.build.prepare_files", {anonymous = true})(jobgraph, target, opt) + elseif job_kind == "link" then + import("private.action.build.link_objects", {anonymous = true})(jobgraph, target, opt) + else + import("private.action.build.build_" .. target:kind(), {anonymous = true})(jobgraph, target, opt) + end + end +end + +-- add target jobs for the given script +function add_targetjobs_for_script(jobgraph, target, instance, opt) + opt = opt or {} + local has_script = false + local buildcmds = opt.buildcmds + local job_prefix = target:fullname() + if target == instance then + job_prefix = job_prefix .. "/target" + else + job_prefix = job_prefix .. "/rule/" .. instance:fullname() + end + + -- call script + if not has_script and not buildcmds then + local script_name = opt.script_name + local script = instance:script(script_name) + if script then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build(function (target, jobgraph, opt) + -- end, {jobgraph = true}) + if instance:extraconf(script_name, "jobgraph") then + script(target, jobgraph) + elseif instance:extraconf(script_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", instance:fullname(), script_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build(function (target, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_name) + jobgraph:add(jobname, function (index, total, opt) + script(target, {progress = opt.progress}) + end) + end + has_script = true + end + end + + -- call command script + -- e.g. + -- + -- target("test") + -- on_buildcmd(function (target, batchcmds, opt) + -- end) + if not has_script then + local scriptcmd_name = opt.scriptcmd_name + local scriptcmd = instance:script(scriptcmd_name) + if scriptcmd then + local jobname = string.format("%s/%s", job_prefix, scriptcmd_name) + jobgraph:add(jobname, function (index, total, opt) + if buildcmds then + -- only generate cmds and do not run them, use cases: e.g. project generator + scriptcmd(target, buildcmds, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd(target, batchcmds_, {progress = opt.progress}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end) + has_script = true + end + end + return has_script +end + +-- add target jobs with the given stage +-- stage: before, after or "" +function add_targetjobs_with_stage(jobgraph, target, stage, opt) + opt = opt or {} + local job_kind = opt.job_kind + local ignored_rules = opt.ignored_rules + + -- the group name, e.g. foo/after_prepare, bar/before_build + local group_name = string.format("%s/%s_%s", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare, before/after_build + local script_name = stage ~= "" and (job_kind .. "_" .. stage) or job_kind + + -- the command script name, e.g. before/after_preparecmd, before/after_buildcmd + local scriptcmd_name = stage ~= "" and (job_kind .. "cmd_" .. stage) or (job_kind .. "cmd") + + -- call target and rules script + local instances = {target} + for _, ruleinst in ipairs(target:orderules()) do + -- we only ignore some builtin rules, so we need not to use fullname. + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + table.insert(instances, ruleinst) + end + end + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local has_script = false + local script_opt = { + script_name = script_name, + scriptcmd_name = scriptcmd_name, + buildcmds = opt.buildcmds + } + for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if add_targetjobs_for_script(jobgraph, target, instance, script_opt) then + has_script = true + end + end) + -- if custom target.on_build/prepare exists, we need to ignore all scripts in rules + if has_script and instance == target and stage == "" then + break + end + end + + -- call builtin script, e.g. on_prepare, on_build, ... + if not has_script and stage == "" then + add_targetjobs_for_builtin_script(jobgraph, target, opt) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) + return group_name +end + +-- add target jobs for the given target +function add_targetjobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + local pkgenvs = _g.pkgenvs + if pkgenvs == nil then + pkgenvs = {} + _g.pkgenvs = pkgenvs + end + + local buildcmds = opt.buildcmds + local job_kind = opt.job_kind + local job_begin = string.format("%s/begin_%s", target:fullname(), job_kind) + local job_end = string.format("%s/end_%s", target:fullname(), job_kind) + jobgraph:add(job_begin, function (index, total, opt) + if buildcmds then + return + end + + -- enter package environments + -- https://github.com/xmake-io/xmake/issues/4033 + -- + -- maybe mixing envs isn't a great solution, + -- but it's the most efficient compromise compared to setting envs in every on_build_file. + -- + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = target:pkgenvs() + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + + -- clean target first if rebuild + if job_kind == "prepare" and target:is_rebuilt() and not option.get("dry-run") then + _clean_target(target) + end + end) + + jobgraph:add(job_end, function (index, total, opt) + if buildcmds then + return + end + + -- restore environments + if target:pkgenvs() then + pkgenvs.oldenvs = pkgenvs.oldenvs or os.getenvs() + pkgenvs.newenvs = pkgenvs.newenvs or {} + pkgenvs.newenvs[target] = nil + local newenvs = pkgenvs.oldenvs + for _, envs in pairs(pkgenvs.newenvs) do + newenvs = os.joinenvs(envs, newenvs) + end + os.setenvs(newenvs) + end + end) + + -- add jobs with target stage, e.g. begin -> before_xxx -> on_xxx -> after_xxx + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end + jobgraph:add_orders(job_begin, group_before, group, group_after, job_end) +end + +-- add target jobs for the given target and deps +function add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_targetjobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_targetjobs_and_deps(jobgraph, dep, targetrefs, opt) + _add_targetjobs_orders(jobgraph, target, dep, opt) + end + end +end + +-- get target jobs +function get_targetjobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_targetjobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add file jobs for the given script +function add_filejobs_for_script(jobgraph, target, instance, sourcebatch, opt) + opt = opt or {} + local has_script = false + local buildcmds = opt.buildcmds + local job_prefix = target:fullname() + local file_group = sourcebatch.rulename + if target == instance then + job_prefix = job_prefix .. "/target/" .. file_group + else + job_prefix = job_prefix .. "/rule/" .. file_group + end + + -- call script files + if not has_script and not buildcmds then + local script_files_name = opt.script_files_name + local script_files = instance:script(script_files_name) + if script_files then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, jobgraph, sourcebatch, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_files_name, "distcc") + if instance:extraconf(script_files_name, "jobgraph") then + script_files(target, jobgraph, sourcebatch, {distcc = distcc}) + elseif instance:extraconf(script_files_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_files_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_files(function (target, sourcebatch, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_files_name) + jobgraph:add(jobname, function (index, total, opt) + script_files(target, sourcebatch, {progress = opt.progress, distcc = distcc}) + end) + end + has_script = true + end + end + + -- call script file + if not has_script and not buildcmds then + local script_file_name = opt.script_file_name + local script_file = instance:script(script_file_name) + if script_file then + -- call custom script with jobgraph + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, jobgraph, sourcefile, opt) + -- end, {jobgraph = true}) + local distcc = instance:extraconf(script_file_name, "distcc") + if instance:extraconf(script_file_name, "jobgraph") then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, jobgraph, sourcefile, {sourcekind = sourcekind, distcc = distcc}) + end + elseif instance:extraconf(script_file_name, "batch") then + wprint("%s.%s: the batch mode is deprecated, please use jobgraph mode instead of it, or disable `build.jobgraph` policy to use it.", + instance:fullname(), script_file_name) + else + -- call custom script directly + -- e.g. + -- + -- target("test") + -- on_build_file(function (target, sourcefile, opt) + -- end) + local jobname = string.format("%s/%s", job_prefix, script_file_name) + jobgraph:add(jobname, function (index, total, opt) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + script_file(target, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + end) + end + has_script = true + end + end + + -- call command script files + -- e.g. + -- + -- target("test") + -- on_buildcmd_files(function (target, batchcmds, sourcebatch, opt) + -- end) + if not has_script then + local scriptcmd_files_name = opt.scriptcmd_files_name + local scriptcmd_files = instance:script(scriptcmd_files_name) + if scriptcmd_files then + local distcc = instance:extraconf(scriptcmd_files_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_files_name) + jobgraph:add(jobname, function (index, total, opt) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + scriptcmd_files(target, buildcmds, sourcebatch, {progress = opt.progress}) + else + local batchcmds_ = batchcmds.new({target = target}) + scriptcmd_files(target, batchcmds_, sourcebatch, {progress = opt.progress, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end) + has_script = true + end + end + + -- call command script file + -- e.g. + -- + -- target("test") + -- on_buildcmd_file(function (target, batchcmds, sourcefile, opt) + -- end) + if not has_script then + local scriptcmd_file_name = opt.scriptcmd_file_name + local scriptcmd_file = instance:script(scriptcmd_file_name) + if scriptcmd_file then + local distcc = instance:extraconf(scriptcmd_file_name, "distcc") + local jobname = string.format("%s/%s", job_prefix, scriptcmd_file_name) + jobgraph:add(jobname, function (index, total, opt) + -- only generate cmds and do not run them, use cases: e.g. project generator + if buildcmds then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, buildcmds, sourcefile, {progress = opt.progress, sourcekind = sourcekind}) + end + else + local batchcmds_ = batchcmds.new({target = target}) + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + scriptcmd_file(target, batchcmds_, sourcefile, {progress = opt.progress, sourcekind = sourcekind, distcc = distcc}) + end + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end + end) + has_script = true + end + end + return has_script +end + +-- add file jobs with the given stage +-- stage: before, after or "" +-- +function add_filejobs_with_stage(jobgraph, target, sourcebatches, stage, opt) + opt = opt or {} + local buildcmds = opt.buildcmds + local ignored_rules = opt.ignored_rules + local job_kind = opt.job_kind + local job_kind_file = job_kind .. "_file" + local job_kind_files = job_kind .. "_files" + local job_kindcmd_file = job_kind .. "cmd_file" + local job_kindcmd_files = job_kind .. "cmd_files" + + -- the group name, e.g. foo/after_prepare_files, bar/before_build_files + local group_name = string.format("%s/%s_%s_files", target:fullname(), stage ~= "" and stage or "on", job_kind) + + -- the script name, e.g. before/after_prepare_files, before/after_build_files + local script_file_name = stage ~= "" and (job_kind_file .. "_" .. stage) or job_kind_file + local script_files_name = stage ~= "" and (job_kind_files .. "_" .. stage) or job_kind_files + + -- the command script name, e.g. before/after_preparecmd_files, before/after_buildcmd_files + local scriptcmd_file_name = stage ~= "" and (job_kindcmd_file .. "_" .. stage) or job_kindcmd_file + local scriptcmd_files_name = stage ~= "" and (job_kindcmd_files .. "_" .. stage) or job_kindcmd_files + + -- build sourcebatches map + local instances = {target} + local sourcebatches_map = {} + local sourcebatches_for_target = {} + for _, sourcebatch in pairs(sourcebatches) do + local rulename = sourcebatch.rulename + if rulename then + -- we only ignore some builtin rules, so we need not to use fullname. + local ruleinst = rule_utils.get_rule(target, rulename) + if not ignored_rules or not ignored_rules:has(ruleinst:name()) then + sourcebatches_map[ruleinst] = sourcebatch + -- avoid duplicate scripts being called twice in the target, + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + if ruleinst:script("build_file") or ruleinst:script("build_files") then + table.insert(sourcebatches_for_target, sourcebatch) + end + table.insert(instances, ruleinst) + end + else + table.insert(sourcebatches_for_target, sourcebatch) + end + end + + -- call target and rules script + local jobsize = jobgraph:size() + jobgraph:group(group_name, function () + local script_opt = { + script_file_name = script_file_name, + script_files_name = script_files_name, + scriptcmd_file_name = scriptcmd_file_name, + scriptcmd_files_name = scriptcmd_files_name, + buildcmds = buildcmds + } + local has_target_script = false + for _, instance in ipairs(instances) do + -- we need to use this group to sort rule scripts with add_orders + local script_group = group_name .. "/" .. instance:fullname() + jobgraph:group(script_group, function () + if instance == target then + for _, sourcebatch in ipairs(sourcebatches_for_target) do + local has_script = add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + -- if custom target.on_build_file[s] exists, we need to ignore all scripts in rules + if has_script and stage == "" then + has_target_script = true + end + end + elseif not has_target_script then -- rule + local sourcebatch = sourcebatches_map[instance] + if sourcebatch then + add_filejobs_for_script(jobgraph, target, instance, sourcebatch, script_opt) + end + end + end) + end + end) + + -- no any new jobs + if jobgraph:size() == jobsize then + return + end + + -- sort build rules + rule_utils.build_orders_in_jobgraph(jobgraph, target, instances, {root_group = group_name}) + return group_name +end + +-- add file jobs for the given target +function add_filejobs(jobgraph, target, opt) + opt = opt or {} + if not target:is_enabled() then + return + end + + -- get sourcebatches + local sourcebatches + local filepatterns = opt.filepatterns + if filepatterns then + sourcebatches = _match_sourcebatches(target, filepatterns) + else + sourcebatches = target:sourcebatches() + end + + -- add file jobs with target stage, e.g. before_xxx_files -> on_xxx_files -> after_xxx_files + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_filejobs_with_stage(jobgraph, target, sourcebatches, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_filejobs_with_stage(jobgraph, target, sourcebatches, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_filejobs_with_stage(jobgraph, target, sourcebatches, "after", opt) + end + jobgraph:add_orders(group_before, group, group_after) +end + +-- add file jobs for the given target and deps +function add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + local targetname = target:fullname() + if not targetrefs[targetname] then + targetrefs[targetname] = target + add_filejobs(jobgraph, target, opt) + for _, depname in ipairs(target:get("deps")) do + local dep = project.target(depname, {namespace = target:namespace()}) + add_filejobs_and_deps(jobgraph, dep, targetrefs, opt) + end + end +end + +-- get files jobs +function get_filejobs(targets_root, opt) + local jobgraph = async_jobgraph.new(opt.job_kind) + local targetrefs = {} + for _, target in ipairs(targets_root) do + add_filejobs_and_deps(jobgraph, target, targetrefs, opt) + end + return jobgraph +end + +-- add link jobs for the given target +function add_linkjobs(jobgraph, target, opt) + opt = table.clone(opt or {}) + opt.job_kind = "link" + local with_stages = opt.with_stages + local group, group_before, group_after + if not with_stages or with_stages:has("on") then + group = add_targetjobs_with_stage(jobgraph, target, "", opt) + end + if not with_stages or with_stages:has("before") then + group_before = add_targetjobs_with_stage(jobgraph, target, "before", opt) + end + if not with_stages or with_stages:has("after") then + group_after = add_targetjobs_with_stage(jobgraph, target, "after", opt) + end + jobgraph:add_orders(group_before, group, group_after) +end + +-- get link depfiles +function get_linkdepfiles(target) + local extrafiles = {} + for _, dep in ipairs(target:orderdeps()) do + if dep:kind() == "static" then + table.insert(extrafiles, dep:targetfile()) + end + end + local linkdepfiles = target:data("linkdepfiles") + if linkdepfiles then + table.join2(extrafiles, linkdepfiles) + end + local objectfiles = target:objectfiles() + local depfiles = objectfiles + if #extrafiles > 0 then + depfiles = table.join(objectfiles, extrafiles) + end + return depfiles +end + +-- get all root targets +function get_root_targets(targetnames, opt) + opt = opt or {} + + -- get root targets + local targets_root = {} + if targetnames then + for _, targetname in ipairs(table.wrap(targetnames)) do + local target = project.target(targetname) + if target then + table.insert(targets_root, target) + if option.get("rebuild") then + target:data_set("rebuilt", true) + if not option.get("shallow") then + for _, dep in ipairs(target:orderdeps()) do + dep:data_set("rebuilt", true) + end + end + end + end + end + else + local group_pattern = opt.group_pattern + local depset = hashset.new() + local targets = {} + for _, target in ipairs(project.ordertargets()) do + if target:is_enabled() then + local group = target:get("group") + if (target:is_default() and not group_pattern) or option.get("all") or (group_pattern and group and group:match(group_pattern)) then + for _, depname in ipairs(target:get("deps")) do + depset:insert(depname) + end + table.insert(targets, target) + end + end + end + for _, target in ipairs(targets) do + if not depset:has(target:name()) then + table.insert(targets_root, target) + end + if option.get("rebuild") then + target:data_set("rebuilt", true) + end + end + end + return targets_root +end + +-- run target-level jobs, e.g. on_prepare, on_build, ... +function run_targetjobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_targetjobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + +-- run files-level jobs, e.g. on_prepare_files, on_build_files, ... +function run_filejobs(targets_root, opt) + opt = opt or {} + local job_kind = opt.job_kind + local jobgraph = get_filejobs(targets_root, opt) + if jobgraph and not jobgraph:empty() then + local curdir = os.curdir() + async_runjobs(job_kind, jobgraph, {on_exit = function (errors) + import("utils.progress") + if errors and progress.showing_without_scroll() then + print("") + end + end, comax = option.get("jobs") or 1, curdir = curdir, distcc = opt.distcc, progress_factor = opt.progress_factor}) + os.cd(curdir) + return true + end +end + diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 87547ab3f..02fac7a3e 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -222,7 +222,7 @@ function _load_require(require_str, requires_extra, opt) -- check require options local extra_options = hashset.of("plat", "arch", "kind", "host", "targetos", "alias", "group", "system", "option", "default", "optional", "debug", - "verify", "external", "private", "build", "configs", "version") + "verify", "external", "private", "build", "configs", "version", "public") for name, value in pairs(require_extra) do if not extra_options:has(name) then wprint("add_requires(\"%s\") has unknown option: {%s=%s}!", require_str, name, tostring(value)) diff --git a/xmake/modules/private/diagnosis/dump_targets.lua b/xmake/modules/private/diagnosis/dump_targets.lua deleted file mode 100644 index c992a28f0..000000000 --- a/xmake/modules/private/diagnosis/dump_targets.lua +++ /dev/null @@ -1,74 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file dump_targets.lua --- - --- imports -import("core.base.hashset") -import("core.project.config") -import("core.project.project") - --- get targets -function _get_targets(targetname) - - -- get targets - local targets = {} - if targetname then - table.insert(targets, project.target(targetname)) - else - for _, target in pairs(project.targets()) do - table.insert(targets, target) - end - end - return targets -end - --- dump the build jobs, e.g. xmake l private.diagnosis.dump_buildjobs [targetname] -function main(targetname) - config.load() - for _, target in ipairs(_get_targets(targetname)) do - cprint("${bright}target(%s):${clear} %s", target:name(), target:kind()) - local deps = target:get("deps") - if deps then - cprint(" ${color.dump.string}deps:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(deps), ", ")) - end - local options = {} - for _, optname in ipairs(target:get("options")) do - if not optname:startswith("__") then - table.insert(options, optname) - end - end - if #options > 0 then - cprint(" ${color.dump.string}options:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(options), ", ")) - end - local packages = target:get("packages") - if packages then - cprint(" ${color.dump.string}packages:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(packages), ", ")) - end - local rules = target:get("rules") - if rules then - cprint(" ${color.dump.string}rules:") - cprint(" ${yellow}->${clear} %s", table.concat(table.wrap(rules), ", ")) - end - print("") - end -end - diff --git a/xmake/modules/private/utils/rule.lua b/xmake/modules/private/utils/rule.lua new file mode 100644 index 000000000..ed6aec451 --- /dev/null +++ b/xmake/modules/private/utils/rule.lua @@ -0,0 +1,72 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file rule.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") + +-- get rule +-- @note we need to get rule from target first, because we maybe will inject and replace builtin rule in target +function get_rule(target, rulename) + local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or + rule.rule(rulename), "unknown rule: %s", rulename) + return ruleinst +end + +-- build rules orders in jobgraph, we need to add rule job with groups +-- +-- like this: +-- @code +-- local root_group = "" +-- for _, ruleinst in ipairs(rules) do +-- local script_group = root_group .. "/" .. ruleinst:fullname() +-- jobgraph:group(script_group, function () +-- jobgraph:add("xxx", function (index, total, opt) +-- -- call rule script +-- end) +-- end) +-- end +-- +function build_orders_in_jobgraph(jobgraph, target, rules, opt) + opt = opt or {} + local root_group = assert(opt.root_group) + for _, ruleinst in ipairs(rules) do + local orders = table.wrap(ruleinst:get("orders")) + if #orders > 0 then + for _, order in ipairs(orders) do + local joborders = {} + for _, rulename in ipairs(order) do + -- we need to use fullname to support namespace + local ruleinst = get_rule(target, rulename) + local script_group = root_group .. "/" .. ruleinst:fullname() + if jobgraph:has(script_group) then + table.insert(joborders, script_group) + end + end + if #joborders > 0 then + jobgraph:add_orders(joborders) + end + end + end + end +end + diff --git a/xmake/plugins/project/clang/compile_commands.lua b/xmake/plugins/project/clang/compile_commands.lua index 46a9bfa64..4cf6eb1ab 100644 --- a/xmake/plugins/project/clang/compile_commands.lua +++ b/xmake/plugins/project/clang/compile_commands.lua @@ -27,7 +27,6 @@ import("core.project.project") import("core.language.language") import("private.utils.batchcmds") import("private.utils.executable_path") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("actions.test.main", {rootdir = os.programdir(), alias = "test_action"}) @@ -242,25 +241,16 @@ end -- add target commands function _add_target_commands(jsonfile, target) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before"}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups) + local cmds_before = target_cmds.get_target_buildcmds(target, {stages = {"before", "on"}}) _add_target_custom_commands(jsonfile, target, "before", cmds_before) -- add target source commands _add_target_source_commands(jsonfile, target) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after"}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after"}) + local cmds_after = target_cmds.get_target_buildcmds(target, {stages = {"after"}}) _add_target_custom_commands(jsonfile, target, "after", cmds_after) end @@ -317,6 +307,7 @@ end function make(outputdir) local oldir = os.cd(os.projectdir()) local jsonfile = io.open(path.join(outputdir, "compile_commands.json"), "w") + target_cmds.prepare_targets() _add_targets(jsonfile) jsonfile:close() os.cd(oldir) diff --git a/xmake/plugins/project/clang/compile_flags.lua b/xmake/plugins/project/clang/compile_flags.lua index 3c76b1bdb..5cfedbb19 100644 --- a/xmake/plugins/project/clang/compile_flags.lua +++ b/xmake/plugins/project/clang/compile_flags.lua @@ -85,10 +85,11 @@ end -- - https://clang.llvm.org/docs/JSONCompilationDatabase.html -- function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- make all local flags = {} flags = _make_all(flags) @@ -100,6 +101,5 @@ function make(outputdir) end flagfile:close() - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 22edabcd2..9d2fc453a 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -29,7 +29,6 @@ import("core.project.rule") import("core.platform.platform") import("lib.detect.find_tool") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("private.utils.target", {alias = "target_utils"}) import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) @@ -1201,9 +1200,6 @@ end -- add target custom commands function _add_target_custom_commands(cmakelists, target, outputdir) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules if _can_native_support_for_cxxmodules() then @@ -1212,17 +1208,12 @@ function _add_target_custom_commands(cmakelists, target, outputdir) -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) end @@ -1331,21 +1322,11 @@ function _generate_cmakelists(cmakelists, outputdir) end end --- make function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) - - -- open the cmakelists local cmakelists = io.open(path.join(outputdir, "CMakeLists.txt"), "w") - - -- generate cmakelists + target_cmds.prepare_targets() _generate_cmakelists(cmakelists, outputdir) - - -- close the cmakelists cmakelists:close() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/make/makefile.lua b/xmake/plugins/project/make/makefile.lua index 3d4d5507f..79277b3f1 100644 --- a/xmake/plugins/project/make/makefile.lua +++ b/xmake/plugins/project/make/makefile.lua @@ -27,7 +27,6 @@ import("core.language.language") import("core.platform.platform") import("lib.detect.find_tool") import("private.utils.batchcmds") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) -- tranlate path @@ -469,14 +468,11 @@ function _add_build_phony(makefile, target) end -- add custom commands before building target -function _add_build_custom_commands_before(makefile, target, sourcegroups, outputdir) +function _add_build_custom_commands_before(makefile, target, outputdir) -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before"}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups) + local cmds_before = target_cmds.get_target_buildcmds(target, {stages = {"before", "on"}}) local targetname = target:name() local label = "precmds_" .. targetname @@ -494,10 +490,8 @@ function _add_build_custom_commands_before(makefile, target, sourcegroups, outpu end -- add custom commands after building target -function _add_build_custom_commands_after(makefile, target, sourcegroups, outputdir) - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after"}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after"}) +function _add_build_custom_commands_after(makefile, target, outputdir) + local cmds_after = target_cmds.get_target_buildcmds(target, {stages = {"after"}}) if #cmds_after > 0 then for _, cmd in ipairs(cmds_after) do local command = _get_command_string(cmd, outputdir) @@ -514,11 +508,8 @@ function _add_build_target(makefile, target, targetflags, outputdir) -- https://github.com/xmake-io/xmake/issues/2337 target:data_set("plugin.project.kind", "makefile") - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- add custom commands before building target - local precmds_label = _add_build_custom_commands_before(makefile, target, sourcegroups, outputdir) + local precmds_label = _add_build_custom_commands_before(makefile, target, outputdir) -- is phony target? if target:is_phony() then @@ -602,7 +593,7 @@ function _add_build_target(makefile, target, targetflags, outputdir) makefile:writef("\t$(VV)%s\n", command) -- add custom commands after building target - _add_build_custom_commands_after(makefile, target, sourcegroups, outputdir) + _add_build_custom_commands_after(makefile, target, outputdir) -- end makefile:print("") @@ -687,10 +678,11 @@ function _add_clean(makefile, outputdir) end function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- open the makefile local makefile = io.open(path.join(outputdir, "makefile"), "w") @@ -715,7 +707,5 @@ function make(outputdir) -- close the makefile makefile:close() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/ninja/build_ninja.lua b/xmake/plugins/project/ninja/build_ninja.lua index 819758447..11be6f4ae 100644 --- a/xmake/plugins/project/ninja/build_ninja.lua +++ b/xmake/plugins/project/ninja/build_ninja.lua @@ -28,6 +28,7 @@ import("core.tool.compiler") import("lib.detect.find_tool") import("lib.detect.find_toolname") import("core.tools.cl.parse_include") +import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) -- this sourcebatch is built? function _sourcebatch_is_built(sourcebatch) @@ -449,10 +450,11 @@ function _add_build_for_targets(ninjafile, outputdir) end function make(outputdir) - - -- enter project directory local oldir = os.cd(os.projectdir()) + -- prepare targets + target_cmds.prepare_targets() + -- open the build.ninja file -- -- we need to change encoding to support msvc_deps_prefix @@ -474,8 +476,6 @@ function make(outputdir) -- close the ninjafile ninjafile:close() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/plugins/project/utils/target_cmds.lua b/xmake/plugins/project/utils/target_cmds.lua index 332e4858d..b18acb943 100644 --- a/xmake/plugins/project/utils/target_cmds.lua +++ b/xmake/plugins/project/utils/target_cmds.lua @@ -24,94 +24,60 @@ import("core.project.config") import("core.base.hashset") import("core.project.rule") import("private.utils.batchcmds") -import("private.utils.rule_groups") +import("private.action.build.target", {alias = "target_buildutils"}) --- this sourcebatch is built? -function _sourcebatch_is_built(sourcebatch) - -- we can only use rulename to filter them because sourcekind may be bound to multiple rules - local rulename = sourcebatch.rulename - if rulename == "c.build" or rulename == "c++.build" - or rulename == "asm.build" or rulename == "cuda.build" - or rulename == "objc.build" or rulename == "objc++.build" - or rulename == "win.sdk.resource" then - return true - end +-- prepare targets +function prepare_targets() + local targets_root = target_buildutils.get_root_targets() + target_buildutils.run_targetjobs(targets_root, {job_kind = "prepare"}) end --- get target buildcmd commands -function get_target_buildcmd(target, cmds, opt) +-- get target buildcmds +function get_target_buildcmds(target, opt) opt = opt or {} - local suffix = opt.suffix - local ignored_rules = hashset.from(opt.ignored_rules or {}) - for _, ruleinst in ipairs(target:orderules()) do - if not ignored_rules:has(ruleinst:name()) then - local scriptname = "buildcmd" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end - end + local progress_wrapper = {} + progress_wrapper.current = function () + return count end -end - --- get target buildcmd_files commands -function get_target_buildcmd_files(target, cmds, sourcebatch, opt) - opt = opt or {} - - -- get rule - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = assert(target:rule(rulename) or project.rule(rulename, {namespace = target:namespace()}) or - rule.rule(rulename), "unknown rule: %s", rulename) - local ignored_rules = hashset.from(opt.ignored_rules or {}) - if ignored_rules:has(ruleinst:name()) then - return + progress_wrapper.total = function () + return total end - - -- generate commands for xx_buildcmd_files - local suffix = opt.suffix - local scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcebatch, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) + progress_wrapper.percent = function () + if total and total > 0 then + return math.floor((count * 100) / total) + else + return 0 end end - - -- generate commands for xx_buildcmd_file - if not script then - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcefile, {}) - if not batchcmds_:empty() then - table.join2(cmds, batchcmds_:cmds()) - end - end + debug.setmetatable(progress_wrapper, { + __tostring = function () + -- we do not output any progress info for the project generators + return "" end - end -end - --- get target buildcmd commands of source group -function get_target_buildcmd_sourcegroups(target, cmds, sourcegroups, opt) - for idx, group in irpairs(sourcegroups) do - for _, item in pairs(group) do - -- buildcmd scripts are always in rule, so we need to ignore target item (item.target). - local sourcebatch = item.sourcebatch - if item.rule then - if not _sourcebatch_is_built(sourcebatch) then - get_target_buildcmd_files(target, cmds, sourcebatch, opt) + }) + local buildcmds = batchcmds.new({target = target}) + local jobgraph = target_buildutils.get_targetjobs({target}, { + job_kind = "build", + buildcmds = buildcmds, + with_stages = hashset.from(opt.stages or {}), + ignored_rules = hashset.from(opt.ignored_rules or {})}) + if jobgraph and not jobgraph:empty() then + local total = jobgraph:size() + local index = 0 + local jobqueue = jobgraph:build() + while true do + local job = jobqueue:getfree() + if job then + if job.run then + job.run(index, total, {progress = progress_wrapper}) end + jobqueue:remove(job) + index = index + 1 + else + break end end end + return buildcmds:cmds() end diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index b6b2e11d7..e3b9ddc95 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -39,7 +39,6 @@ import("private.action.require.install", {alias = "install_requires"}) import("private.action.run.runenvs") import("actions.config.configfiles", {alias = "generate_configfiles", rootdir = os.programdir()}) import("private.utils.batchcmds") -import("private.utils.rule_groups") import("plugins.project.utils.target_cmds", {rootdir = os.programdir()}) function _translate_path(dir, vcxprojdir) @@ -132,24 +131,18 @@ function _make_custom_commands(target, vcxprojdir) return _translate_path(p, vcxprojdir) end) - -- build sourcebatch groups first - local sourcegroups = rule_groups.build_sourcebatch_groups(target, target:sourcebatches()) - -- ignore c++ modules rules local ignored_rules = _get_cxxmodules_rules() -- add before commands -- we use irpairs(groups), because the last group that should be given the highest priority. - local cmds_before = {} - target_cmds.get_target_buildcmd(target, cmds_before, {suffix = "before", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {suffix = "before", ignored_rules = ignored_rules}) - -- rule.on_buildcmd_files should also be executed before building the target, as cmake PRE_BUILD does not work. - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_before, sourcegroups, {ignored_rules = ignored_rules}) + -- rule.on_buildcmd_files should also be executed before building the target + local cmds_before = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"before", "on"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "before", cmds_before) -- add after commands - local cmds_after = {} - target_cmds.get_target_buildcmd_sourcegroups(target, cmds_after, sourcegroups, {suffix = "after", ignored_rules = ignored_rules}) - target_cmds.get_target_buildcmd(target, cmds_after, {suffix = "after", ignored_rules = ignored_rules}) + local cmds_after = target_cmds.get_target_buildcmds(target, {ignored_rules = ignored_rules, stages = {"after"}}) + _add_target_custom_commands_for_batchcmds(cmakelists, target, outputdir, "after", cmds_after) local commands = {} for _, cmd in ipairs(cmds_before) do @@ -386,17 +379,14 @@ end -- make vstudio project function make(outputdir, vsinfo) - - -- enter project directory local oldir = os.cd(project.directory()) - -- init solution directory - vsinfo.solution_dir = path.join(outputdir, "vs" .. vsinfo.vstudio_version) + -- prepare targets + target_cmds.prepare_targets() - -- init modes + -- init vsinfo + vsinfo.solution_dir = path.join(outputdir, "vs" .. vsinfo.vstudio_version) vsinfo.modes = _make_vsinfo_modes() - - -- init archs vsinfo.archs = _make_vsinfo_archs() -- load targets @@ -519,7 +509,5 @@ function make(outputdir, vsinfo) -- clear local cache _clear_cache() - - -- leave project directory os.cd(oldir) end diff --git a/xmake/rules/asm/xmake.lua b/xmake/rules/asm/xmake.lua index 78b693137..1c6743793 100644 --- a/xmake/rules/asm/xmake.lua +++ b/xmake/rules/asm/xmake.lua @@ -21,7 +21,7 @@ -- define rule: asm.build rule("asm.build") set_sourcekinds("as") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: asm rule("asm") diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a71e2c392..7ec120cc9 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -44,10 +44,15 @@ function _build_modules(target, sourcebatch, modules, opt) cppfile = cppfile or module.cppfile local deps = {} - for _, dep in ipairs(table.keys(module.requires or {})) do - table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + for name, req in pairs(module.requires or {}) do + -- we need to use the full path as dep name if requre item is headerunit + local dep = name + if req.method:startswith("include-") and req.path then + dep = path.normalize(req.path) + end + local depname = target:fullname() .. "/module/" .. dep + table.insert(deps, depname) end - opt.build_module(deps, module, name, objectfile, cppfile) ::continue:: @@ -56,7 +61,6 @@ end -- build target headerunits function _build_headerunits(target, headerunits, opt) - local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true}) if opt.stl_headerunit then outputdir = path.join(outputdir, "stl") @@ -70,11 +74,9 @@ function _build_headerunits(target, headerunits, opt) local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) local key = path.normalize(headerunit.path) local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) - if build then mark_build(target, key) end - opt.build_headerunit(headerunit, key, bmifile, outputdir, build) end end @@ -128,7 +130,7 @@ function _try_reuse_modules(target, modules) end local mapped = get_from_target_mapper(dep, name) if mapped then - compiler_support.memcache():set2(target:name() .. name, "reuse", true) + compiler_support.memcache():set2(target:fullname() .. name, "reuse", true) add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) break end @@ -156,8 +158,8 @@ function should_build(target, sourcefile, bmifile, opt) for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) if m then - local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) - or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key) + local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) + or compiler_support.memcache():get2("should_build_in_" .. target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -172,7 +174,7 @@ function should_build(target, sourcefile, bmifile, opt) if opt.name then local m = get_from_target_mapper(target, opt.name) if m and m.opt and m.opt.target then - local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) if rebuild then dependinfo.files = {} table.insert(dependinfo.files, sourcefile) @@ -204,7 +206,6 @@ end -- "file": "foo.cppm" -- } function _generate_meta_module_info(target, name, sourcefile, requires) - local modulehash = compiler_support.get_modulehash(target, sourcefile) local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} @@ -223,7 +224,7 @@ end function _target_module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end function _is_duplicated_headerunit(target, key) @@ -251,7 +252,7 @@ function _builder(target) end function mark_build(target, name) - compiler_support.memcache():set2("should_build_in_" .. target:name(), name, true) + compiler_support.memcache():set2("should_build_in_" .. target:fullname(), name, true) end -- build batchjobs for modules @@ -262,11 +263,11 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob}) -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "_populate_module_map" + local populate_jobname = target:fullname() .. "/module/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -277,20 +278,49 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, objectfile, cppfile) - local job_name = name and target:name() .. name or cppfile - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, - {module = module, objectfile = objectfile, cppfile = cppfile}) - end + build_module = function(deps, module, name, objectfile, cppfile) + local job_name = target:fullname() .. "/module/" .. (name or cppfile) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, + {module = module, objectfile = objectfile, cppfile = cppfile}) + end })) -- build batchjobs for modules build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) end +-- build modules for jobgraph +function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local jobdeps = {} + local jobsize = jobgraph:size() + local build_modules_group = target:fullname() .. "/module/build_modules" + jobgraph:group(build_modules_group, function () + + -- add populate module job + local populate_jobname = target:fullname() .. "/module/populate_module_map" + jobgraph:add(populate_jobname, function(index, total, opt) + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + end) + + -- add module jobs + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, module, name, objectfile, cppfile) + local jobname = target:fullname() .. "/module/" .. (name or cppfile) + _builder(target).make_module_jobgraph(target, jobgraph, { + module = module, objectfile = objectfile, cppfile = cppfile + }) + jobdeps[jobname] = table.join(populate_jobname, deps) + end}) + ) + end) + if jobgraph:size() > jobsize then + return build_modules_group, jobdeps + end +end + -- build modules for batchcmds function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local depmtime = 0 opt.progress = opt.progress or 0 @@ -299,15 +329,16 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, module, _, objectfile, cppfile) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) - end + build_module = function(_, module, _, objectfile, cppfile) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, { + module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + end })) batchcmds:set_depmtime(depmtime) end --- generate headerunits for batchjobs +-- build headerunits for batchjobs function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -318,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob}) local build_headerunits = function(headerunits) local modulesjobs = {} _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:name() .. key + local job_name = target:fullname() .. "/module/" .. key local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job @@ -345,9 +376,46 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules end end --- generate headerunits for batchcmds -function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) +-- build headerunits for jobgraph +function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + local jobsize = jobgraph:size() + local build_headerunits_group = target:fullname() .. "/module/build_headerunits" + jobgraph:group(build_headerunits_group, function () + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:fullname() .. "/module/" .. key + _builder(target).make_headerunit_jobgraph(target, + job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + end + })) + end + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end + end) + if jobgraph:size() > jobsize then + return build_headerunits_group + end +end + +-- build headerunits for batchcmds +function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) if not user_headerunits and not stl_headerunits then return @@ -374,6 +442,31 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end +-- build modules and headerunits, and we need to build headerunits first +function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) + if jobgraph.add_orders then + local build_modules_group, jobdeps = build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local build_headerunits_group = build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + if build_modules_group then + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end + if build_headerunits_group then + jobgraph:add_orders(build_headerunits_group, build_modules_group) + end + end + elseif jobgraph.runcmds then + build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) + elseif jobgraph.newjob then -- deprecated + build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) + end +end + +-- generate metadata function generate_metadata(target, modules) local public_modules for _, module in table.orderpairs(modules) do @@ -391,11 +484,11 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) - progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name) local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) json.savefile(metafilepath, metadata) end, {comax = jobs, total = #public_modules}) @@ -404,20 +497,20 @@ end -- flush target module mapper keys function flush_target_module_mapper_keys(target) local memcache = compiler_support.memcache() - memcache:set2(target:name(), "module_mapper_keys", nil) + memcache:set2(target:fullname(), "module_mapper_keys", nil) end -- get or create a target module mapper function get_target_module_mapper(target) local memcache = compiler_support.memcache() - local mapper = memcache:get2(target:name(), "module_mapper") + local mapper = memcache:get2(target:fullname(), "module_mapper") if not mapper then mapper = {} - memcache:set2(target:name(), "module_mapper", mapper) + memcache:set2(target:fullname(), "module_mapper", mapper) end -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit - local mapper_keys = memcache:get2(target:name(), "module_mapper_keys") + local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys") if not mapper_keys then mapper_keys = {} for _, item in pairs(mapper) do @@ -425,7 +518,7 @@ function get_target_module_mapper(target) mapper_keys[item.key] = item end end - memcache:set2(target:name(), "module_mapper_keys", mapper_keys) + memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys) end return mapper, mapper_keys end @@ -463,7 +556,7 @@ end -- check if dependencies changed function is_dependencies_changed(target, module) - local cachekey = target:name() .. module.name + local cachekey = target:fullname() .. module.name local requires = hashset.from(table.keys(module.requires or {})) local oldrequires = compiler_support.memcache():get2(cachekey, "oldrequires") local changed = false @@ -481,3 +574,44 @@ function is_dependencies_changed(target, module) end return requires, changed end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch, opt) + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + end + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) + end + end + + -- patch objectfiles and dependencies + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + + local dependfile = target:dependfile(sourcefile or objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 972694254..ec6692c35 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -134,7 +134,7 @@ function _get_requiresflags(target, module, opt) local modulefileflag = compiler_support.get_modulefileflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -216,11 +216,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -263,11 +263,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) end else @@ -278,6 +278,75 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local is_mapped_bmi = mapped_bmi ~= nil + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -285,7 +354,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -305,11 +374,11 @@ function make_module_buildcmds(target, batchcmds, opt) local is_mapped_bmi = mapped_bmi ~= nil if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds, is_mapped_bmi = is_mapped_bmi}) end else @@ -340,7 +409,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local depvalues = {compinst:program(), compflags} if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end @@ -351,6 +420,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show(jobopt.progress, + "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", + target:fullname(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) @@ -358,7 +458,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 06b522ce8..715e8a07d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -36,7 +36,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index cf64a08bb..eed30a320 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -69,20 +69,6 @@ function strip_flags(target, flags) return _compiler_support(target).strip_flags(target, flags) end --- patch sourcebatch -function patch_sourcebatch(target, sourcebatch) - sourcebatch.sourcekind = "cxx" - sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - table.insert(sourcebatch.objectfiles, objectfile) - - local dependfile = target:dependfile(sourcefile or objectfile) - table.insert(sourcebatch.dependfiles, dependfile) - end -end - -- get bmi extension function get_bmi_extension(target) return _compiler_support(target).get_bmi_extension() @@ -224,7 +210,7 @@ function modules_cachedir(target, opt) end function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:name() + local key = path.directory(modulepath) .. target:fullname() return hash.uuid(key):split("-", {plain = true})[1]:lower() end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 9d552d0fd..94b8a4804 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -202,9 +202,8 @@ function _get_edges(nodes, modules) return edges end -function _get_package_modules(target, package, opt) +function _get_package_modules(target, package) local package_modules - local modulesdir = path.join(package:installdir(), "modules") local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) for _, metafile in ipairs(metafiles) do @@ -213,42 +212,39 @@ function _get_package_modules(target, package, opt) local moduleonly = not package:libraryfiles() package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata, external = {moduleonly = moduleonly}} end - return package_modules end --- generate dependency files -function _generate_dependencies(target, sourcebatch, opt) - local changed = false - if opt.batchjobs then - local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_module_dependency_scanner", function(index) - local sourcefile = sourcebatch.sourcefiles[index] - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end, {comax = jobs, total = #sourcebatch.sourcefiles}) - else - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed - end - end - return changed -end --- get module dependencies -function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:name() .. "/" .. sourcebatch.rulename - local modules = compiler_support.memcache():get2("modules", cachekey) - if modules == nil then - modules = compiler_support.localcache():get2("modules", cachekey) - opt.progress = opt.progress or 0 - local changed = _generate_dependencies(target, sourcebatch, opt) - if changed or modules == nil then +-- generate module dependencies +function generate_module_dependencies(target, jobgraph, sourcebatch, opt) + local parsejob = target:fullname() .. "/parse_module_dependencies" + jobgraph:add(parsejob, function (index, total, opt) + local changed = compiler_support.memcache():get2("modules", "dependencies_changed") + if changed then + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) - modules = _parse_dependencies_data(target, moduleinfos) + local modules = _parse_dependencies_data(target, moduleinfos) compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() end - compiler_support.memcache():set2("modules", cachekey, modules) + end) + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local jobname = target:fullname() .. "/generate_module_dependencies/" .. sourcefile + jobgraph:add(jobname, function (index, total, opt) + local changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) + if changed then + compiler_support.memcache():set2("modules", "dependencies_changed", true) + end + end) + jobgraph:add_orders(jobname, parsejob) end +end + +-- get module dependencies +function get_module_dependencies(target, sourcebatch) + local cachekey = target:fullname() .. "/" .. sourcebatch.rulename + local modules = compiler_support.localcache():get2("modules", cachekey) + assert(modules, "no module dependencies!") return modules end @@ -387,7 +383,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end -- extract packages modules dependencies -function get_all_packages_modules(target, opt) +function get_all_packages_modules(target) -- parse all meta-info and append their informations to the package store local packages = target:pkgs() or {} @@ -397,7 +393,7 @@ function get_all_packages_modules(target, opt) local packages_modules for _, package in table.orderpairs(packages) do - local package_modules = _get_package_modules(target, package, opt) + local package_modules = _get_package_modules(target, package) if package_modules then packages_modules = packages_modules or {} table.join2(packages_modules, package_modules) @@ -415,19 +411,21 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) end - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, objectfile in ipairs(cycle) do - local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + local objectfiles_sorted, has_cycle = dag:topo_sort() + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, objectfile in ipairs(cycle) do + local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + table.insert(names, name or cppfile) + end + local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) table.insert(names, name or cppfile) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) - table.insert(names, name or cppfile) - raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - - local objectfiles_sorted = table.reverse(dag:topological_sort()) + objectfiles_sorted = table.reverse(objectfiles_sorted) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then @@ -465,7 +463,7 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) end end end - if insert then + if insert then table.insert(build_objectfiles, objectfile) table.insert(link_objectfiles, objectfile) elseif external and not external.from_moduleonly then @@ -474,8 +472,8 @@ function sort_modules_by_dependencies(target, objectfiles, modules, opt) objectfiles_sorted_set:remove(objectfile) if name ~= "std" and name ~= "std.compat" then culleds = culleds or {} - culleds[target:name()] = culleds[target:name()] or {} - table.insert(culleds[target:name()], format("%s -> %s", name, cppfile)) + culleds[target:fullname()] = culleds[target:fullname()] or {} + table.insert(culleds[target:fullname()], format("%s -> %s", name, cppfile)) end end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index a0a43db5a..4033eefd5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -76,7 +76,7 @@ end function _module_map_cachekey(target) local mode = config.mode() - return target:name() .. "module_mapper" .. (mode or "") + return target:fullname() .. "module_mapper" .. (mode or "") end -- generate a module mapper file for build a headerunit @@ -135,7 +135,7 @@ end -- function _generate_modulemapper_file(target, module, cppfile) local maplines = _get_maplines(target, module) - local mapper_path = path.join(os.tmpdir(), target:name():replace(" ", "_"), name or cppfile:replace(" ", "_")) + local mapper_path = path.join(os.tmpdir(), target:fullname():replace(" ", "_"), name or cppfile:replace(" ", "_")) local mapper_content = {} table.insert(mapper_content, "root " .. path.unix(os.projectdir())) for _, mapline in ipairs(maplines) do @@ -182,11 +182,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -217,13 +217,13 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -241,6 +241,70 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) + sourcefile = opt.cppfile + end + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + if sourcefile then + _compile(target, flags, sourcefile, opt.objectfile) + end + os.tryrm(module_mapper) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -249,7 +313,7 @@ function make_module_buildcmds(target, batchcmds, opt) local module_mapperflag = compiler_support.get_modulemapperflag(target) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -272,13 +336,13 @@ function make_module_buildcmds(target, batchcmds, opt) local sourcefile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) local module_onlyflag = compiler_support.get_moduleonlyflag(target) table.insert(flags, module_onlyflag) sourcefile = opt.cppfile end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) sourcefile = opt.cppfile end if option.get("diagnosis") then @@ -297,7 +361,6 @@ end -- build headerunit file for batchjobs function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local _headerunit = headerunit _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) @@ -320,7 +383,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end @@ -337,9 +400,48 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) + if option.get("diagnosis") then + print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _compile(target, + _make_headerunitflags(target, headerunit, headerunit_mapper, opt), + path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + + + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) batchcmds:mkdir(outputdir) @@ -349,7 +451,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) if option.get("diagnosis") then batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) end diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 11dab5669..e087c6d4e 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) depend.on_changed(function() if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end local outputdir = compiler_support.get_outputdir(target, sourcefile) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 5998db62f..8f582438b 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -167,7 +167,7 @@ function _get_requiresflags(target, module, opt) local headerunitflag = compiler_support.get_headerunitflag(target) local name = module.name - local cachekey = target:name() .. name + local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") @@ -175,7 +175,7 @@ function _get_requiresflags(target, module, opt) local deps_flags = {} for required in requires:orderitems() do local dep_module = get_from_target_mapper(target, required) - assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) + assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:fullname()) local mapflag local bmifile = dep_module.bmi @@ -257,19 +257,18 @@ end -- build module file for batchjobs function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/module/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -311,11 +310,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) end else @@ -326,6 +325,72 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -333,7 +398,7 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local mapped_bmi - if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if provide and compiler_support.memcache():get2(target:fullname() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi end @@ -353,11 +418,11 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = mapped_bmi or bmifile if external and not from_moduleonly then if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_bmi_step(target, bmifile, opt.cppfile, provide, {batchcmds = batchcmds}) end else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) end else @@ -390,7 +455,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif local name = headerunit.unique and headerunit.name or headerunit.path if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end @@ -401,6 +466,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) @@ -408,7 +504,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), target:objectfile(headerunit.path)) end batchcmds:add_depfiles(headerunit.path) diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index 491a27cbf..044510fab 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -40,7 +40,7 @@ function generate_dependency_for(target, sourcefile, opt) local changed = false depend.on_changed(function () - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) local outputdir = compiler_support.get_outputdir(target, sourcefile) local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index f967dd133..a27633182 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -37,7 +37,7 @@ rule("c++.build.modules") -- even if some sub-targets do not contain C++ modules. -- -- maybe we will have a more fine-grained configuration strategy to disable it in the future. - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- disable ccache for this target -- @@ -70,66 +70,48 @@ rule("c++.build.modules.builder") set_sourcekinds("cxx") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") - -- parallel build support to accelerate `xmake build` to build modules - before_build_files(function(target, batchjobs, sourcebatch, opt) + -- generate module dependencies + on_prepare_files(function (target, jobgraph, sourcebatch, opt) if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") import("modules_support.builder") + import("modules_support.dependency_scanner") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end + -- patch sourcebatch + builder.patch_sourcebatch(target, sourcebatch) - opt = opt or {} - opt.batchjobs = true + -- generate module dependencies + dependency_scanner.generate_module_dependencies(target, jobgraph, sourcebatch, opt) + end + end, {jobgraph = true}) - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- parallel build support to accelerate `xmake build` to build modules + before_build_files(function(target, jobgraph, sourcebatch, opt) + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build modules - builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - - -- build headerunits and we need to do it before building modules - builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - + -- build modules and headerunits + builder.build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end - end, {batch = true}) + end, {jobgraph = true, batch = true}) -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) @@ -138,54 +120,22 @@ rule("c++.build.modules.builder") import("modules_support.dependency_scanner") import("modules_support.builder") - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) - end - end - - opt = opt or {} - opt.batchjobs = false - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - + -- get module dependencies + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch) if not target:is_moduleonly() then -- avoid building non referenced modules local build_objectfiles, link_objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) sourcebatch.objectfiles = build_objectfiles - -- build headerunits - builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - - -- build modules - builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - + -- build headerunits and modules + builder.build_modules_and_headerunits(target, batchcmds, sourcebatch, modules, opt) sourcebatch.objectfiles = link_objectfiles else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end - compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():set2(target:fullname(), "c++.modules", modules) compiler_support.localcache():save() else sourcebatch.sourcefiles = {} @@ -222,7 +172,7 @@ rule("c++.build.modules.install") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets if compiler_support.contains_modules(target) then - local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + local modules = compiler_support.localcache():get2(target:fullname(), "c++.modules") builder.generate_metadata(target, modules) compiler_support.add_installfiles_for_modules(target) diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 7216a1f13..30ab31106 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -22,15 +22,16 @@ rule("c.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "c", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "c", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "c", opt) + end, {jobgraph = true}) rule("c++.build.pcheader") + add_orders("c++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "cxx", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "cxx", opt) + end, {jobgraph = true}) diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua index 94bba431e..e04cd5140 100644 --- a/xmake/rules/c++/xmake.lua +++ b/xmake/rules/c++/xmake.lua @@ -21,18 +21,8 @@ rule("c.build") set_sourcekinds("cc") add_deps("c.build.pcheader", "c.build.optimization", "c.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) - -- enable vs runtime as MD by default - if target:is_plat("windows") and not target:get("runtimes") then - local vs_runtime_default = target:policy("build.c++.msvc.runtime") - if vs_runtime_default and target:has_tool("cc", "cl", "clang_cl") then - if is_mode("debug") then - vs_runtime_default = vs_runtime_default .. "d" - end - target:set("runtimes", vs_runtime_default) - end - end -- https://github.com/xmake-io/xmake/issues/4621 if target:is_plat("windows") and target:is_static() and target:has_tool("cc", "tcc") then target:set("extension", ".a") @@ -43,22 +33,12 @@ rule("c.build") rule("c++.build") set_sourcekinds("cxx") add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization", "c++.build.sanitizer") - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) on_config(function (target) -- enable c++ exceptions by default if target:is_plat("windows") and not target:get("exceptions") then target:set("exceptions", "cxx") end - -- enable vs runtime as MD by default - if target:is_plat("windows") and not target:get("runtimes") then - local vs_runtime_default = target:policy("build.c++.msvc.runtime") - if vs_runtime_default and target:has_tool("cxx", "cl", "clang_cl") then - if is_mode("debug") then - vs_runtime_default = vs_runtime_default .. "d" - end - target:set("runtimes", vs_runtime_default) - end - end -- https://github.com/xmake-io/xmake/issues/4621 if target:is_plat("windows") and target:is_static() and target:has_tool("cxx", "tcc") then target:set("extension", ".a") diff --git a/xmake/rules/cppfront/xmake.lua b/xmake/rules/cppfront/xmake.lua index 22663f184..772c59487 100644 --- a/xmake/rules/cppfront/xmake.lua +++ b/xmake/rules/cppfront/xmake.lua @@ -22,7 +22,6 @@ rule("cppfront.build.h2") set_extensions(".h2") on_buildcmd_file(function (target, batchcmds, sourcefile_h2, opt) - -- get cppfront import("lib.detect.find_tool") local cppfront = assert(find_tool("cppfront", {check = "-h"}), "cppfront not found!") @@ -42,12 +41,11 @@ rule("cppfront.build.h2") batchcmds:set_depcache(target:dependfile(sourcefile_h)) end) --- define rule: cppfront.build rule("cppfront.build.cpp2") set_extensions(".cpp2") -- .h2 must compile before .cpp2 - add_deps("cppfront.build.h2", {order = true}) + add_orders("cppfront.build.h2", "cppfront.build.cpp2") on_load(function (target) -- only cppfront source files? we need to patch cxx source kind for linker @@ -64,7 +62,6 @@ rule("cppfront.build.cpp2") end end) on_buildcmd_file(function (target, batchcmds, sourcefile_cpp2, opt) - -- get cppfront import("lib.detect.find_tool") local cppfront = assert(find_tool("cppfront", {check = "-h"}), "cppfront not found!") @@ -84,6 +81,7 @@ rule("cppfront.build.cpp2") batchcmds:add_depfiles(path.join(root_dir, match_h2)) end end + -- add commands local argv = {"-o", path(sourcefile_cpp), path(sourcefile_cpp2)} batchcmds:show_progress(opt.progress, "${color.build.object}compiling.cpp2 %s", sourcefile_cpp2) @@ -97,14 +95,9 @@ rule("cppfront.build.cpp2") batchcmds:set_depcache(target:dependfile(objectfile)) end) - -- define rule: cppfront rule("cppfront") - - -- add_build.h2 rules add_deps("cppfront.build.h2") - - -- add build rules add_deps("cppfront.build.cpp2") -- set compiler runtime, e.g. vs runtime diff --git a/xmake/rules/cuda/xmake.lua b/xmake/rules/cuda/xmake.lua index 334f569f1..4ce6699ff 100644 --- a/xmake/rules/cuda/xmake.lua +++ b/xmake/rules/cuda/xmake.lua @@ -22,7 +22,7 @@ rule("cuda.build") set_sourcekinds("cu") add_deps("cuda.build.devlink") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- https://github.com/xmake-io/xmake/issues/4755 local cu_ccbin = target:tool("cu-ccbin") diff --git a/xmake/rules/dlang/xmake.lua b/xmake/rules/dlang/xmake.lua index 7ffb8c42f..f5e741f7a 100644 --- a/xmake/rules/dlang/xmake.lua +++ b/xmake/rules/dlang/xmake.lua @@ -21,7 +21,7 @@ rule("dlang.build") set_sourcekinds("dc") add_deps("dlang.build.optimization") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_load(function (target) local toolchains = target:get("toolchains") or get_config("toolchain") if not toolchains or not table.contains(table.wrap(toolchains), "dlang", "dmd", "ldc", "gdc") then diff --git a/xmake/rules/fortran/xmake.lua b/xmake/rules/fortran/xmake.lua index 4ab11906d..d10993a91 100644 --- a/xmake/rules/fortran/xmake.lua +++ b/xmake/rules/fortran/xmake.lua @@ -35,7 +35,7 @@ rule("fortran.build") add_deps("fortran.build.modules") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) end) on_build_files(function (target, sourcebatch, opt) import("private.action.build.object").build(target, sourcebatch, opt) diff --git a/xmake/rules/go/xmake.lua b/xmake/rules/go/xmake.lua index 58890e260..fd4cdfb4a 100644 --- a/xmake/rules/go/xmake.lua +++ b/xmake/rules/go/xmake.lua @@ -23,7 +23,7 @@ rule("go.build") add_deps("go.env") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- xxx.a if target:is_static() then target:set("prefixname", "") diff --git a/xmake/rules/lex_yacc/lex/xmake.lua b/xmake/rules/lex_yacc/lex/xmake.lua index 3ee16b7f4..d57f06036 100644 --- a/xmake/rules/lex_yacc/lex/xmake.lua +++ b/xmake/rules/lex_yacc/lex/xmake.lua @@ -21,7 +21,7 @@ -- define rule: lex rule("lex") add_deps("c++") - add_deps("yacc", {order = true}) + add_orders("yacc", "lex") set_extensions(".l", ".ll") before_buildcmd_file(function (target, batchcmds, sourcefile_lex, opt) diff --git a/xmake/rules/objc++/precompiled_header/xmake.lua b/xmake/rules/objc++/precompiled_header/xmake.lua index db2d9f036..9f2b21ade 100644 --- a/xmake/rules/objc++/precompiled_header/xmake.lua +++ b/xmake/rules/objc++/precompiled_header/xmake.lua @@ -22,15 +22,16 @@ rule("objc.build.pcheader") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "m", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "m", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "m", opt) + end, {jobgraph = true}) rule("objc++.build.pcheader") + add_orders("objc++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "mxx", opt) end) - before_build(function (target, opt) - import("private.action.build.pcheader").build(target, "mxx", opt) - end) + before_prepare(function (target, jobgraph, opt) + import("private.action.build.pcheader").build(target, jobgraph, "mxx", opt) + end, {jobgraph = true}) diff --git a/xmake/rules/objc++/xmake.lua b/xmake/rules/objc++/xmake.lua index c653261e7..3714ddf3f 100644 --- a/xmake/rules/objc++/xmake.lua +++ b/xmake/rules/objc++/xmake.lua @@ -31,7 +31,7 @@ rule("objc.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc++.build rule("objc++.build") @@ -46,7 +46,7 @@ rule("objc++.build") target:add("frameworks", "Foundation", "CoreFoundation") end end) - on_build_files("private.action.build.object", {batch = true, distcc = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true, distcc = true}) -- define rule: objc rule("objc++") diff --git a/xmake/rules/platform/windows/idl/xmake.lua b/xmake/rules/platform/windows/idl/xmake.lua index 75c47f7e9..8cffe18df 100644 --- a/xmake/rules/platform/windows/idl/xmake.lua +++ b/xmake/rules/platform/windows/idl/xmake.lua @@ -22,10 +22,13 @@ rule("platform.windows.idl") set_extensions(".idl") - on_config(function (target) - local autogendir = path.join(target:autogendir(), "platform/windows/idl") - os.mkdir(autogendir) - target:add("includedirs", autogendir, {public = true}) + on_config("windows", "mingw", function (target) + local sourcebatch = target:sourcebatches()["platform.windows.idl"] + if sourcebatch then + local autogendir = path.join(target:autogendir(), "platform/windows/idl") + os.mkdir(autogendir) + target:add("includedirs", autogendir, {public = true}) + end end) before_buildcmd_file(function (target, batchcmds, sourcefile, opt) diff --git a/xmake/rules/protobuf/proto.lua b/xmake/rules/protobuf/proto.lua index c35ffb697..bebeb3105 100644 --- a/xmake/rules/protobuf/proto.lua +++ b/xmake/rules/protobuf/proto.lua @@ -22,10 +22,8 @@ import("core.base.option") import("lib.detect.find_tool") import("core.project.depend") -import("private.action.build.object", {alias = "build_objectfiles"}) import("utils.progress") import("private.utils.batchcmds") -import("private.async.buildjobs") -- get protoc function _get_protoc(target, sourcekind) @@ -100,7 +98,7 @@ function load(target, sourcekind) end end -function buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) +function buildcmd_pfile(target, batchcmds, sourcefile_proto, sourcekind, opt) -- get protoc local protoc = _get_protoc(target, sourcekind) @@ -166,7 +164,7 @@ function buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) end end -function buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) +function buildcmd_cxfile(target, batchcmds, sourcefile_proto, sourcekind, opt) -- get protoc local protoc = _get_protoc(target, sourcekind) @@ -223,90 +221,3 @@ function buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, sourcekind) end end -function build_cxfile_objects(target, batchjobs, opt, sourcekind) - local sourcebatch_cx = { - rulename = (sourcekind == "cxx" and "c++" or "c").. ".build", - sourcekind = sourcekind, - sourcefiles = {}, - objectfiles = {}, - dependfiles = {} - } - for _, sourcefile_proto in ipairs(sourcefiles) do - -- get c/c++ source file for protobuf - local prefixdir - local autogendir - local public - local grpc_cpp_plugin - local fileconfig = target:fileconfig(sourcefile_proto) - if fileconfig then - public = fileconfig.proto_public - prefixdir = fileconfig.proto_rootdir - -- custom autogen directory to access the generated header files - -- @see https://github.com/xmake-io/xmake/issues/3678 - autogendir = fileconfig.proto_autogendir - grpc_cpp_plugin = fileconfig.proto_grpc_cpp_plugin - end - local rootdir = autogendir and autogendir or path.join(target:autogendir(), "rules", "protobuf") - local filename = path.basename(sourcefile_proto) .. ".pb" .. (sourcekind == "cxx" and ".cc" or "-c.c") - local sourcefile_cx = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename}) - local sourcefile_dir = prefixdir and path.join(rootdir, prefixdir) or path.directory(sourcefile_cx) - - local grpc_cpp_plugin_bin - local filename_grpc - local sourcefile_cx_grpc - if grpc_cpp_plugin then - grpc_cpp_plugin_bin = _get_grpc_cpp_plugin(target, sourcekind) - filename_grpc = path.basename(sourcefile_proto) .. ".grpc.pb.cc" - sourcefile_cx_grpc = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename_grpc}) - end - - -- add includedirs - target:add("includedirs", sourcefile_dir, {public = public}) - - -- add objectfile - local objectfile = target:objectfile(sourcefile_cx) - local dependfile = target:dependfile(sourcefile_proto) - table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx) - table.insert(sourcebatch_cx.objectfiles, objectfile) - table.insert(sourcebatch_cx.dependfiles, dependfile) - - local objectfile_grpc - if grpc_cpp_plugin then - objectfile_grpc = target:objectfile(sourcefile_cx_grpc) - table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx_grpc) - table.insert(sourcebatch_cx.objectfiles, objectfile_grpc) - table.insert(sourcebatch_cx.dependfiles, dependfile) - end - end - build_objectfiles(target, batchjobs, sourcebatch_cx, opt) -end - --- build batch jobs -function build_cxfiles(target, batchjobs, sourcebatch, opt, sourcekind) - opt = opt or {} - local nodes = {} - local nodenames = {} - local node_rulename = "rules/" .. sourcebatch.rulename .. "/node" - local sourcefiles = sourcebatch.sourcefiles - for _, sourcefile_proto in ipairs(sourcefiles) do - local nodename = node_rulename .. "/" .. sourcefile_proto - nodes[nodename] = { - name = nodename, - job = batchjobs:addjob(nodename, function(index, total, jobopt) - local batchcmds_ = batchcmds.new({target = target}) - buildcmd_pfiles(target, batchcmds_, sourcefile_proto, {progress = jobopt.progress}, sourcekind) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end) - } - table.insert(nodenames, nodename) - end - local rootname = "rules/" .. sourcebatch.rulename .. "/root" - nodes[rootname] = { - name = rootname, - deps = nodenames, - job = batchjobs:addjob(rootname, function(_index, _total) - build_cxfile_objects(target, batchjobs, opt, sourcekind) - end) - } - buildjobs(nodes, batchjobs, opt.rootjob) -end diff --git a/xmake/rules/protobuf/xmake.lua b/xmake/rules/protobuf/xmake.lua index 119ff3d1f..f2cbf2615 100644 --- a/xmake/rules/protobuf/xmake.lua +++ b/xmake/rules/protobuf/xmake.lua @@ -27,14 +27,11 @@ rule("protobuf.cpp") end) -- generate build commands before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cxx") + import("proto").buildcmd_pfile(target, batchcmds, sourcefile_proto, "cxx", opt) end) on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cxx") + import("proto").buildcmd_cxfile(target, batchcmds, sourcefile_proto, "cxx", opt) end) - before_build_files(function (target, batchjobs, sourcebatch, opt) - import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cxx") - end, {batch = true}) -- define rule: protobuf.c @@ -45,11 +42,8 @@ rule("protobuf.c") import("proto").load(target, "cc") end) before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cc") + import("proto").buildcmd_pfile(target, batchcmds, sourcefile_proto, "cc", opt) end) on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt) - import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cc") + import("proto").buildcmd_cxfile(target, batchcmds, sourcefile_proto, "cc", opt) end) - before_build_files(function (target, batchjobs, sourcebatch, opt) - import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cc") - end, {batch = true}) diff --git a/xmake/rules/qt/moc/xmake.lua b/xmake/rules/qt/moc/xmake.lua index 2807f2bc9..4ece876cf 100644 --- a/xmake/rules/qt/moc/xmake.lua +++ b/xmake/rules/qt/moc/xmake.lua @@ -20,7 +20,7 @@ rule("qt.moc") add_deps("qt.env") - add_deps("qt.ui", {order = true}) + add_orders("qt.ui", "qt.moc") set_extensions(".h", ".hpp") before_buildcmd_file(function (target, batchcmds, sourcefile, opt) import("core.tool.compiler") diff --git a/xmake/rules/swift/xmake.lua b/xmake/rules/swift/xmake.lua index d1ea0e886..9a2dfbb5b 100644 --- a/xmake/rules/swift/xmake.lua +++ b/xmake/rules/swift/xmake.lua @@ -21,7 +21,7 @@ -- define rule: swift.build rule("swift.build") set_sourcekinds("sc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) on_config(function (target) -- we use swift-frontend to support multiple modules -- @see https://github.com/xmake-io/xmake/issues/3916 diff --git a/xmake/rules/utils/bin2c/xmake.lua b/xmake/rules/utils/bin2c/xmake.lua index 5d0dbfd31..392b2f20a 100644 --- a/xmake/rules/utils/bin2c/xmake.lua +++ b/xmake/rules/utils/bin2c/xmake.lua @@ -20,6 +20,7 @@ rule("utils.bin2c") set_extensions(".bin") + add_orders("utils.bin2c", "c++.build.modules.builder") on_load(function (target) local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c") if not os.isdir(headerdir) then @@ -27,7 +28,7 @@ rule("utils.bin2c") end target:add("includedirs", headerdir) end) - before_buildcmd_file(function (target, batchcmds, sourcefile_bin, opt) + on_preparecmd_file(function (target, batchcmds, sourcefile_bin, opt) -- get header file local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c") diff --git a/xmake/rules/utils/compiler_runtime/xmake.lua b/xmake/rules/utils/compiler_runtime/xmake.lua index 1a50fddef..cd8d5297c 100644 --- a/xmake/rules/utils/compiler_runtime/xmake.lua +++ b/xmake/rules/utils/compiler_runtime/xmake.lua @@ -40,5 +40,16 @@ rule("utils.compiler.runtime") end target:set("runtimes", runtimes) end + + -- enable vs runtime as MD by default + if target:is_plat("windows") and not target:get("runtimes") then + local vs_runtime_default = target:policy("build.c++.msvc.runtime") + if vs_runtime_default and target:has_tool("cxx", "cl", "clang_cl") then + if is_mode("debug") then + vs_runtime_default = vs_runtime_default .. "d" + end + target:set("runtimes", vs_runtime_default) + end + end end) diff --git a/xmake/rules/vala/xmake.lua b/xmake/rules/vala/xmake.lua index caebab9d8..e2262a9ec 100644 --- a/xmake/rules/vala/xmake.lua +++ b/xmake/rules/vala/xmake.lua @@ -26,7 +26,7 @@ rule("vala.build") set_sourcekinds("cc") on_load(function (target) -- we disable to build across targets in parallel, because the source files may depend on other target modules - target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.fence", true) -- get vapi file local vapifile = target:data("vala.vapifile") diff --git a/xmake/rules/verilator/verilator.lua b/xmake/rules/verilator/verilator.lua index 6b7267861..0a87c40ff 100644 --- a/xmake/rules/verilator/verilator.lua +++ b/xmake/rules/verilator/verilator.lua @@ -207,7 +207,7 @@ endmodule]]) os.rm(tmpdir) end -function build_cppfiles(target, batchjobs, sourcebatch, opt) +function build_cppfiles(target, jobgraph, sourcebatch, opt) local toolchain = assert(target:toolchain("verilator"), 'we need to set_toolchains("verilator") in target("%s")', target:name()) local verilator = assert(toolchain:config("verilator"), "verilator not found!") local autogendir = path.join(target:autogendir(), "rules", "verilator") @@ -261,7 +261,7 @@ function build_cppfiles(target, batchjobs, sourcebatch, opt) table.insert(sourcebatch_cpp.objectfiles, objectfile) table.insert(sourcebatch_cpp.dependfiles, dependfile) end - build_objectfiles(target, batchjobs, sourcebatch_cpp, opt) + build_objectfiles(target, jobgraph, sourcebatch_cpp, opt) end function buildcmd_vfiles(target, batchcmds, sourcebatch, opt) diff --git a/xmake/rules/verilator/xmake.lua b/xmake/rules/verilator/xmake.lua index 4e6f49397..2042d8233 100644 --- a/xmake/rules/verilator/xmake.lua +++ b/xmake/rules/verilator/xmake.lua @@ -34,9 +34,9 @@ rule("verilator.binary") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) @@ -61,9 +61,9 @@ rule("verilator.static") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) @@ -88,9 +88,9 @@ rule("verilator.shared") -- Just to avoid before_buildcmd_files being executed at build time end) - on_build_files(function (target, batchjobs, sourcebatch, opt) - import("verilator").build_cppfiles(target, batchjobs, sourcebatch, opt) - end, {batch = true, distcc = true}) + on_build_files(function (target, jobgraph, sourcebatch, opt) + import("verilator").build_cppfiles(target, jobgraph, sourcebatch, opt) + end, {jobgraph = true, batch = true, distcc = true}) before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) import("verilator").buildcmd_vfiles(target, batchcmds, sourcebatch, opt) diff --git a/xmake/rules/winsdk/xmake.lua b/xmake/rules/winsdk/xmake.lua index 0544abb86..6461c4eca 100644 --- a/xmake/rules/winsdk/xmake.lua +++ b/xmake/rules/winsdk/xmake.lua @@ -21,7 +21,7 @@ -- define rule: win.sdk.resource rule("win.sdk.resource") set_sourcekinds("mrc") - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: application rule("win.sdk.application") diff --git a/xmake/rules/xcode/application/xmake.lua b/xmake/rules/xcode/application/xmake.lua index b4969b3df..7d5edc668 100644 --- a/xmake/rules/xcode/application/xmake.lua +++ b/xmake/rules/xcode/application/xmake.lua @@ -27,19 +27,6 @@ rule("xcode.application") -- we must set kind before target.on_load(), may we will use target in on_load() on_load("load") - -- depend xcode.framework? we need to disable `build.across_targets_in_parallel` policy - after_load(function (target) - local across_targets_in_parallel - for _, dep in ipairs(target:orderdeps()) do - if dep:rule("xcode.framework") then - across_targets_in_parallel = false - end - end - if across_targets_in_parallel ~= nil then - target:set("policy", "build.across_targets_in_parallel", across_targets_in_parallel) - end - end) - -- build *.app after_build("build") diff --git a/xmake/rules/xcode/bundle/xmake.lua b/xmake/rules/xcode/bundle/xmake.lua index 31c0a8296..972d7d75e 100644 --- a/xmake/rules/xcode/bundle/xmake.lua +++ b/xmake/rules/xcode/bundle/xmake.lua @@ -62,7 +62,7 @@ rule("xcode.bundle") end end) - after_build(function (target, opt) + after_link(function (target, opt) -- imports import("core.base.option") diff --git a/xmake/rules/xcode/framework/xmake.lua b/xmake/rules/xcode/framework/xmake.lua index 10b2f337f..46850c019 100644 --- a/xmake/rules/xcode/framework/xmake.lua +++ b/xmake/rules/xcode/framework/xmake.lua @@ -78,7 +78,7 @@ rule("xcode.framework") end end) - after_build(function (target, opt) + after_link(function (target, opt) -- imports import("core.base.option") diff --git a/xmake/rules/zig/xmake.lua b/xmake/rules/zig/xmake.lua index 4565fcda6..3d0a0a4b9 100644 --- a/xmake/rules/zig/xmake.lua +++ b/xmake/rules/zig/xmake.lua @@ -26,7 +26,7 @@ rule("zig.build") os.mkdir(cachedir) target:add("zcflags", "--cache-dir " .. cachedir) end) - on_build_files("private.action.build.object", {batch = true}) + on_build_files("private.action.build.object", {jobgraph = true, batch = true}) -- define rule: zig rule("zig") |
